From 8a99eba96d7db6031e0c4a6ae5d8f8a842572a8d Mon Sep 17 00:00:00 2001 From: Akira Ajisaka Date: Wed, 26 Apr 2017 15:29:53 +0900 Subject: [PATCH] Revert "YARN-5007. Remove deprecated constructors of MiniYARNCluster and MiniMRYarnCluster. Contributed by Andras Bokor." This reverts commit 34ab8e73d48ea3c21c9e9571419b43b605de15c3. --- .../mapred/TestMRTimelineEventHandling.java | 2 +- .../mapreduce/v2/MiniMRYarnCluster.java | 6 +++- .../yarn/client/ProtocolHATestBase.java | 6 ++-- .../hadoop/yarn/server/MiniYARNCluster.java | 22 ++++++++++-- .../yarn/server/TestMiniYarnCluster.java | 34 +++++++++++++++++-- 5 files changed, 61 insertions(+), 9 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java index eca51fb31c..cbca3c8cdb 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java @@ -203,7 +203,7 @@ public void testMRNewTimelineServiceEventHandling() throws Exception { MiniMRYarnCluster cluster = null; try { cluster = new MiniMRYarnCluster( - TestMRTimelineEventHandling.class.getSimpleName(), 1); + TestMRTimelineEventHandling.class.getSimpleName(), 1, true); cluster.init(conf); cluster.start(); LOG.info("A MiniMRYarnCluster get start."); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java index bcfc963512..06e37dd0f0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java @@ -74,7 +74,11 @@ public MiniMRYarnCluster(String testName) { } public MiniMRYarnCluster(String testName, int noOfNMs) { - super(testName, 1, noOfNMs, 4, 4); + this(testName, noOfNMs, false); + } + @Deprecated + public MiniMRYarnCluster(String testName, int noOfNMs, boolean enableAHS) { + super(testName, 1, noOfNMs, 4, 4, enableAHS); historyServerWrapper = new JobHistoryServerWrapper(); addService(historyServerWrapper); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java index c185abca44..a8e91323a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java @@ -274,7 +274,7 @@ protected void startHACluster(int numOfNMs, boolean overrideClientRMService, conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false); cluster = new MiniYARNClusterForHATesting(TestRMFailover.class.getName(), 2, - numOfNMs, 1, 1, overrideClientRMService, overrideRTS, + numOfNMs, 1, 1, false, overrideClientRMService, overrideRTS, overrideApplicationMasterService); cluster.resetStartFailoverFlag(false); cluster.init(conf); @@ -304,10 +304,10 @@ public class MiniYARNClusterForHATesting extends MiniYARNCluster { public MiniYARNClusterForHATesting(String testName, int numResourceManagers, int numNodeManagers, int numLocalDirs, - int numLogDirs, boolean overrideClientRMService, + int numLogDirs, boolean enableAHS, boolean overrideClientRMService, boolean overrideRTS, boolean overrideApplicationMasterService) { super(testName, numResourceManagers, numNodeManagers, numLocalDirs, - numLogDirs); + numLogDirs, enableAHS); this.overrideClientRMService = overrideClientRMService; this.overrideRTS = overrideRTS; this.overrideApplicationMasterService = overrideApplicationMasterService; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java index 974ef093c1..77240c63f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java @@ -146,6 +146,7 @@ public class MiniYARNCluster extends CompositeService { private int numLocalDirs; // Number of nm-log-dirs per nodemanager private int numLogDirs; + private boolean enableAHS; /** * @param testName name of the test @@ -153,13 +154,16 @@ public class MiniYARNCluster extends CompositeService { * @param numNodeManagers the number of node managers in the cluster * @param numLocalDirs the number of nm-local-dirs per nodemanager * @param numLogDirs the number of nm-log-dirs per nodemanager + * @param enableAHS enable ApplicationHistoryServer or not */ + @Deprecated public MiniYARNCluster( String testName, int numResourceManagers, int numNodeManagers, - int numLocalDirs, int numLogDirs) { + int numLocalDirs, int numLogDirs, boolean enableAHS) { super(testName.replace("$", "")); this.numLocalDirs = numLocalDirs; this.numLogDirs = numLogDirs; + this.enableAHS = enableAHS; String testSubDir = testName.replace("$", ""); File targetWorkDir = new File("target", testSubDir); try { @@ -209,6 +213,20 @@ public MiniYARNCluster( nodeManagers = new NodeManager[numNodeManagers]; } + /** + * @param testName name of the test + * @param numResourceManagers the number of resource managers in the cluster + * @param numNodeManagers the number of node managers in the cluster + * @param numLocalDirs the number of nm-local-dirs per nodemanager + * @param numLogDirs the number of nm-log-dirs per nodemanager + */ + public MiniYARNCluster( + String testName, int numResourceManagers, int numNodeManagers, + int numLocalDirs, int numLogDirs) { + this(testName, numResourceManagers, numNodeManagers, numLocalDirs, + numLogDirs, false); + } + /** * @param testName name of the test * @param numNodeManagers the number of node managers in the cluster @@ -270,7 +288,7 @@ public void serviceInit(Configuration conf) throws Exception { } if(conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED) || enableAHS) { addService(new ApplicationHistoryServerWrapper()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java index 92f63eaa58..ff7fafc200 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java @@ -34,14 +34,18 @@ public void testTimelineServiceStartInMiniCluster() throws Exception { int numNodeManagers = 1; int numLocalDirs = 1; int numLogDirs = 1; + boolean enableAHS; /* * Timeline service should not start if TIMELINE_SERVICE_ENABLED == false + * and enableAHS flag == false */ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false); + enableAHS = false; try (MiniYARNCluster cluster = new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(), - numNodeManagers, numLocalDirs, numLogDirs, numLogDirs)) { + numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, + enableAHS)) { cluster.init(conf); cluster.start(); @@ -53,11 +57,14 @@ public void testTimelineServiceStartInMiniCluster() throws Exception { /* * Timeline service should start if TIMELINE_SERVICE_ENABLED == true + * and enableAHS == false */ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); + enableAHS = false; try (MiniYARNCluster cluster = new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(), - numNodeManagers, numLocalDirs, numLogDirs, numLogDirs)) { + numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, + enableAHS)) { cluster.init(conf); // Verify that the timeline-service starts on ephemeral ports by default @@ -67,6 +74,29 @@ public void testTimelineServiceStartInMiniCluster() throws Exception { cluster.start(); + //Timeline service may sometime take a while to get started + int wait = 0; + while(cluster.getApplicationHistoryServer() == null && wait < 20) { + Thread.sleep(500); + wait++; + } + //verify that the timeline service is started. + Assert.assertNotNull("Timeline Service should have been started", + cluster.getApplicationHistoryServer()); + } + /* + * Timeline service should start if TIMELINE_SERVICE_ENABLED == false + * and enableAHS == true + */ + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false); + enableAHS = true; + try (MiniYARNCluster cluster = + new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(), + numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, + enableAHS)) { + cluster.init(conf); + cluster.start(); + //Timeline service may sometime take a while to get started int wait = 0; while(cluster.getApplicationHistoryServer() == null && wait < 20) {