From a4f2995b9ec8347612b7aeeb5a3a8b7191278790 Mon Sep 17 00:00:00 2001 From: Jian He Date: Mon, 15 Dec 2014 11:04:09 -0800 Subject: [PATCH] Revert "YARN-2890. MiniYARNCluster should start the timeline server based on the configuration. Contributed by Mit Desai." This reverts commit 51af8d367de94689770f57c64bea3b244d7755f6. Conflicts: hadoop-yarn-project/CHANGES.txt --- .../TestJobHistoryEventHandler.java | 2 +- .../mapred/TestMRTimelineEventHandling.java | 58 +------------------ .../mapreduce/v2/MiniMRYarnCluster.java | 6 +- hadoop-yarn-project/CHANGES.txt | 3 - .../TestDistributedShell.java | 2 +- .../yarn/client/ProtocolHATestBase.java | 6 +- .../hadoop/yarn/server/MiniYARNCluster.java | 24 +++++++- 7 files changed, 34 insertions(+), 67 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java index a64f1d6a1a..de35d840b9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java @@ -453,7 +453,7 @@ public void testTimelineEventHandling() throws Exception { long currentTime = System.currentTimeMillis(); try { yarnCluster = new MiniYARNCluster( - TestJobHistoryEventHandler.class.getSimpleName(), 1, 1, 1, 1); + TestJobHistoryEventHandler.class.getSimpleName(), 1, 1, 1, 1, true); yarnCluster.init(conf); yarnCluster.start(); jheh.start(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java index 76a3598835..346953f90b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java @@ -34,58 +34,6 @@ public class TestMRTimelineEventHandling { - @Test - public void testTimelineServiceStartInMiniCluster() throws Exception { - Configuration conf = new YarnConfiguration(); - - /* - * Timeline service should not start if the config is set to false - * Regardless to the value of MAPREDUCE_JOB_EMIT_TIMELINE_DATA - */ - conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false); - conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, true); - MiniMRYarnCluster cluster = null; - try { - cluster = new MiniMRYarnCluster( - TestJobHistoryEventHandler.class.getSimpleName(), 1); - cluster.init(conf); - cluster.start(); - - //verify that the timeline service is not started. - Assert.assertNull("Timeline Service should not have been started", - cluster.getApplicationHistoryServer()); - - //Run a MR job and verify it succeeds - Path inDir = new Path("input"); - Path outDir = new Path("output"); - RunningJob job = - UtilsForTests.runJobSucceed(new JobConf(conf), inDir, outDir); - Assert.assertEquals(JobStatus.SUCCEEDED, - job.getJobStatus().getState().getValue()); - } - finally { - if(cluster != null) { - cluster.stop(); - } - } - conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false); - conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, false); - cluster = null; - try { - cluster = new MiniMRYarnCluster( - TestJobHistoryEventHandler.class.getSimpleName(), 1); - cluster.init(conf); - cluster.start(); - Assert.assertNull("Timeline Service should not have been started", - cluster.getApplicationHistoryServer()); - } - finally { - if(cluster != null) { - cluster.stop(); - } - } - } - @Test public void testMRTimelineEventHandling() throws Exception { Configuration conf = new YarnConfiguration(); @@ -94,7 +42,7 @@ public void testMRTimelineEventHandling() throws Exception { MiniMRYarnCluster cluster = null; try { cluster = new MiniMRYarnCluster( - TestJobHistoryEventHandler.class.getSimpleName(), 1); + TestJobHistoryEventHandler.class.getSimpleName(), 1, true); cluster.init(conf); cluster.start(); TimelineStore ts = cluster.getApplicationHistoryServer() @@ -148,7 +96,7 @@ public void testMapreduceJobTimelineServiceEnabled() MiniMRYarnCluster cluster = null; try { cluster = new MiniMRYarnCluster( - TestJobHistoryEventHandler.class.getSimpleName(), 1); + TestJobHistoryEventHandler.class.getSimpleName(), 1, true); cluster.init(conf); cluster.start(); TimelineStore ts = cluster.getApplicationHistoryServer() @@ -185,7 +133,7 @@ public void testMapreduceJobTimelineServiceEnabled() cluster = null; try { cluster = new MiniMRYarnCluster( - TestJobHistoryEventHandler.class.getSimpleName(), 1); + TestJobHistoryEventHandler.class.getSimpleName(), 1, true); cluster.init(conf); cluster.start(); TimelineStore ts = cluster.getApplicationHistoryServer() diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java index 45b736e37d..47b38a1365 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java @@ -72,7 +72,11 @@ public MiniMRYarnCluster(String testName) { } public MiniMRYarnCluster(String testName, int noOfNMs) { - super(testName, 1, noOfNMs, 4, 4); + this(testName, noOfNMs, false); + } + + public MiniMRYarnCluster(String testName, int noOfNMs, boolean enableAHS) { + super(testName, 1, noOfNMs, 4, 4, enableAHS); historyServerWrapper = new JobHistoryServerWrapper(); addService(historyServerWrapper); } diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 6e74d149bb..4edf8a1fbf 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -187,9 +187,6 @@ Release 2.7.0 - UNRELEASED YARN-2905. AggregatedLogsBlock page can infinitely loop if the aggregated log file is corrupted (Varun Saxena via jlowe) - YARN-2890. MiniYARNCluster should start the timeline server based on the - configuration. (Mit Desai via zjshen) - YARN-2894. Fixed a bug regarding application view acl when RM fails over. (Rohith Sharmaks via jianhe) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java index a05b3b0d0a..1d3a1040cf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java @@ -84,7 +84,7 @@ protected void setupInternal(int numNodeManager) throws Exception { if (yarnCluster == null) { yarnCluster = new MiniYARNCluster(TestDistributedShell.class.getSimpleName(), 1, - numNodeManager, 1, 1); + numNodeManager, 1, 1, true); yarnCluster.init(conf); yarnCluster.start(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java index 4b2b4bcc43..da7d50529a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java @@ -295,7 +295,7 @@ protected void startHACluster(int numOfNMs, boolean overrideClientRMService, conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false); cluster = new MiniYARNClusterForHATesting(TestRMFailover.class.getName(), 2, - numOfNMs, 1, 1, overrideClientRMService, overrideRTS, + numOfNMs, 1, 1, false, overrideClientRMService, overrideRTS, overrideApplicationMasterService); cluster.resetStartFailoverFlag(false); cluster.init(conf); @@ -326,10 +326,10 @@ public class MiniYARNClusterForHATesting extends MiniYARNCluster { public MiniYARNClusterForHATesting(String testName, int numResourceManagers, int numNodeManagers, int numLocalDirs, - int numLogDirs, boolean overrideClientRMService, + int numLogDirs, boolean enableAHS, boolean overrideClientRMService, boolean overrideRTS, boolean overrideApplicationMasterService) { super(testName, numResourceManagers, numNodeManagers, numLocalDirs, - numLogDirs); + numLogDirs, enableAHS); this.overrideClientRMService = overrideClientRMService; this.overrideRTS = overrideRTS; this.overrideApplicationMasterService = overrideApplicationMasterService; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java index 2eb5034eab..e83d6014bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java @@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer; import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.MemoryApplicationHistoryStore; +import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService; import org.apache.hadoop.yarn.server.nodemanager.NodeManager; @@ -119,6 +120,7 @@ public class MiniYARNCluster extends CompositeService { private int numLocalDirs; // Number of nm-log-dirs per nodemanager private int numLogDirs; + private boolean enableAHS; /** * @param testName name of the test @@ -126,13 +128,15 @@ public class MiniYARNCluster extends CompositeService { * @param numNodeManagers the number of node managers in the cluster * @param numLocalDirs the number of nm-local-dirs per nodemanager * @param numLogDirs the number of nm-log-dirs per nodemanager + * @param enableAHS enable ApplicationHistoryServer or not */ public MiniYARNCluster( String testName, int numResourceManagers, int numNodeManagers, - int numLocalDirs, int numLogDirs) { + int numLocalDirs, int numLogDirs, boolean enableAHS) { super(testName.replace("$", "")); this.numLocalDirs = numLocalDirs; this.numLogDirs = numLogDirs; + this.enableAHS = enableAHS; String testSubDir = testName.replace("$", ""); File targetWorkDir = new File("target", testSubDir); try { @@ -182,6 +186,20 @@ public MiniYARNCluster( nodeManagers = new NodeManager[numNodeManagers]; } + /** + * @param testName name of the test + * @param numResourceManagers the number of resource managers in the cluster + * @param numNodeManagers the number of node managers in the cluster + * @param numLocalDirs the number of nm-local-dirs per nodemanager + * @param numLogDirs the number of nm-log-dirs per nodemanager + */ + public MiniYARNCluster( + String testName, int numResourceManagers, int numNodeManagers, + int numLocalDirs, int numLogDirs) { + this(testName, numResourceManagers, numNodeManagers, numLocalDirs, + numLogDirs, false); + } + /** * @param testName name of the test * @param numNodeManagers the number of node managers in the cluster @@ -242,8 +260,8 @@ public void serviceInit(Configuration conf) throws Exception { addService(new NodeManagerWrapper(index)); } - if(conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false)) { - addService(new ApplicationHistoryServerWrapper()); + if (enableAHS) { + addService(new ApplicationHistoryServerWrapper()); } super.serviceInit(