From 7e73cad97419671b1020906bef36ad60aaa70ab9 Mon Sep 17 00:00:00 2001 From: Masatake Iwasaki Date: Tue, 30 Jun 2020 10:52:25 +0900 Subject: [PATCH] MAPREDUCE-7280. MiniMRYarnCluster has hard-coded timeout waiting to start history server, with no way to disable. (#2065) --- .../hadoop/mapred/TestMiniMRBringup.java | 24 +++++++++++-- .../mapreduce/v2/MiniMRYarnCluster.java | 34 +++++++++++++------ 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRBringup.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRBringup.java index b608d756a4..fc49fa5690 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRBringup.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRBringup.java @@ -18,9 +18,13 @@ package org.apache.hadoop.mapred; +import java.io.IOException; + +import org.junit.Assert; import org.junit.Test; -import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster; /** * A Unit-test to test bringup and shutdown of Mini Map-Reduce Cluster. @@ -36,5 +40,21 @@ public void testBringUp() throws IOException { if (mr != null) { mr.shutdown(); } } } - + + @Test + public void testMiniMRYarnClusterWithoutJHS() throws IOException { + MiniMRYarnCluster mr = null; + try { + final Configuration conf = new Configuration(); + conf.setBoolean(MiniMRYarnCluster.MR_HISTORY_MINICLUSTER_ENABLED, false); + mr = new MiniMRYarnCluster("testMiniMRYarnClusterWithoutJHS"); + mr.init(conf); + mr.start(); + Assert.assertEquals(null, mr.getHistoryServer()); + } finally { + if (mr != null) { + mr.stop(); + } + } + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java index 299383d76b..dbd87e2491 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java @@ -69,6 +69,10 @@ public class MiniMRYarnCluster extends MiniYARNCluster { private JobHistoryServer historyServer; private JobHistoryServerWrapper historyServerWrapper; private static final String TIMELINE_AUX_SERVICE_NAME = "timeline_collector"; + public static final String MR_HISTORY_MINICLUSTER_ENABLED = + JHAdminConfig.MR_HISTORY_PREFIX + "minicluster.enabled"; + public static final String MR_HISTORY_MINICLUSTER_LAUNCH_TIMEOUT_MS = + JHAdminConfig.MR_HISTORY_PREFIX + "minicluster.launch.timeout.ms"; public MiniMRYarnCluster(String testName) { this(testName, 1); @@ -77,11 +81,10 @@ public MiniMRYarnCluster(String testName) { public MiniMRYarnCluster(String testName, int noOfNMs) { this(testName, noOfNMs, false); } + @Deprecated public MiniMRYarnCluster(String testName, int noOfNMs, boolean enableAHS) { super(testName, 1, noOfNMs, 4, 4, enableAHS); - historyServerWrapper = new JobHistoryServerWrapper(); - addService(historyServerWrapper); } public static String getResolvedMRHistoryWebAppURLWithoutScheme( @@ -118,6 +121,11 @@ public static String getResolvedMRHistoryWebAppURLWithoutScheme( @Override public void serviceInit(Configuration conf) throws Exception { + if (conf.getBoolean(MR_HISTORY_MINICLUSTER_ENABLED, true)) { + historyServerWrapper = new JobHistoryServerWrapper(); + addService(historyServerWrapper); + } + conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); String stagingDir = conf.get(MRJobConfig.MR_AM_STAGING_DIR); if (stagingDir == null || @@ -212,11 +220,13 @@ public void serviceInit(Configuration conf) throws Exception { protected void serviceStart() throws Exception { super.serviceStart(); - //need to do this because historyServer.init creates a new Configuration - getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS, - historyServer.getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS)); - MRWebAppUtil.setJHSWebappURLWithoutScheme(getConfig(), - MRWebAppUtil.getJHSWebappURLWithoutScheme(historyServer.getConfig())); + if (historyServer != null) { + //need to do this because historyServer.init creates a new Configuration + getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS, + historyServer.getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS)); + MRWebAppUtil.setJHSWebappURLWithoutScheme(getConfig(), + MRWebAppUtil.getJHSWebappURLWithoutScheme(historyServer.getConfig())); + } LOG.info("MiniMRYARN ResourceManager address: " + getConfig().get(YarnConfiguration.RM_ADDRESS)); @@ -233,7 +243,6 @@ private class JobHistoryServerWrapper extends AbstractService { public JobHistoryServerWrapper() { super(JobHistoryServerWrapper.class.getName()); } - private volatile boolean jhsStarted = false; @Override public synchronized void serviceStart() throws Exception { @@ -255,12 +264,15 @@ public synchronized void serviceStart() throws Exception { new Thread() { public void run() { historyServer.start(); - jhsStarted = true; }; }.start(); - GenericTestUtils.waitFor(() -> jhsStarted, 1500, 60_000); - + final int launchTimeout = getConfig().getInt( + MR_HISTORY_MINICLUSTER_LAUNCH_TIMEOUT_MS, 60_000); + GenericTestUtils.waitFor( + () -> historyServer.getServiceState() == STATE.STARTED + || historyServer.getServiceState() == STATE.STOPPED, + 100, launchTimeout); if (historyServer.getServiceState() != STATE.STARTED) { throw new IOException("HistoryServer failed to start"); }