From 054270cc28b1d56953851501abebcb3f62372813 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Mon, 10 Dec 2012 18:07:20 +0000 Subject: [PATCH] MAPREDUCE-4703. Add the ability to start the MiniMRClientCluster using the configurations used before it is being stopped. (ahmed.radwan via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1419618 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../hadoop/mapred/MiniMRClientCluster.java | 5 ++ .../mapred/MiniMRClientClusterFactory.java | 4 ++ .../mapred/MiniMRYarnClusterAdapter.java | 25 ++++++++ .../mapred/TestMiniMRClientCluster.java | 61 +++++++++++++++++++ 5 files changed, 98 insertions(+) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index bad1eae0ba..a2c0ebd043 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -171,6 +171,9 @@ Release 2.0.3-alpha - Unreleased MAPREDUCE-4723. Fix warnings found by findbugs 2. (Sandy Ryza via eli) + MAPREDUCE-4703. Add the ability to start the MiniMRClientCluster using + the configurations used before it is being stopped. (ahmed.radwan via tucu) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientCluster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientCluster.java index dc4687b580..eda104e1d7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientCluster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientCluster.java @@ -31,6 +31,11 @@ public interface MiniMRClientCluster { public void start() throws IOException; + /** + * Stop and start back the cluster using the same configuration. + */ + public void restart() throws IOException; + public void stop() throws IOException; public Configuration getConfig() throws IOException; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientClusterFactory.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientClusterFactory.java index f26ace1870..105d364621 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientClusterFactory.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientClusterFactory.java @@ -67,6 +67,10 @@ public static MiniMRClientCluster create(Class caller, int noOfNMs, MiniMRYarnCluster miniMRYarnCluster = new MiniMRYarnCluster(caller .getName(), noOfNMs); + job.getConfiguration().set("minimrclientcluster.caller.name", + caller.getName()); + job.getConfiguration().setInt("minimrclientcluster.nodemanagers.number", + noOfNMs); miniMRYarnCluster.init(job.getConfiguration()); miniMRYarnCluster.start(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRYarnClusterAdapter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRYarnClusterAdapter.java index 81329a97c3..74ef06f791 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRYarnClusterAdapter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRYarnClusterAdapter.java @@ -18,8 +18,13 @@ package org.apache.hadoop.mapred; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster; +import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.service.Service.STATE; /** * An adapter for MiniMRYarnCluster providing a MiniMRClientCluster interface. @@ -29,6 +34,8 @@ public class MiniMRYarnClusterAdapter implements MiniMRClientCluster { private MiniMRYarnCluster miniMRYarnCluster; + private static final Log LOG = LogFactory.getLog(MiniMRYarnClusterAdapter.class); + public MiniMRYarnClusterAdapter(MiniMRYarnCluster miniMRYarnCluster) { this.miniMRYarnCluster = miniMRYarnCluster; } @@ -48,4 +55,22 @@ public void stop() { miniMRYarnCluster.stop(); } + @Override + public void restart() { + if (!miniMRYarnCluster.getServiceState().equals(STATE.STARTED)){ + LOG.warn("Cannot restart the mini cluster, start it first"); + return; + } + Configuration oldConf = new Configuration(getConfig()); + String callerName = oldConf.get("minimrclientcluster.caller.name", + this.getClass().getName()); + int noOfNMs = oldConf.getInt("minimrclientcluster.nodemanagers.number", 1); + oldConf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true); + oldConf.setBoolean(JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, true); + stop(); + miniMRYarnCluster = new MiniMRYarnCluster(callerName, noOfNMs); + miniMRYarnCluster.init(oldConf); + miniMRYarnCluster.start(); + } + } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClientCluster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClientCluster.java index 27e6666cc5..d988c08a6f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClientCluster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClientCluster.java @@ -32,6 +32,8 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -91,6 +93,65 @@ public static void cleanup() throws IOException { mrCluster.stop(); } + @Test + public void testRestart() throws Exception { + + String rmAddress1 = mrCluster.getConfig().get(YarnConfiguration.RM_ADDRESS); + String rmAdminAddress1 = mrCluster.getConfig().get( + YarnConfiguration.RM_ADMIN_ADDRESS); + String rmSchedAddress1 = mrCluster.getConfig().get( + YarnConfiguration.RM_SCHEDULER_ADDRESS); + String rmRstrackerAddress1 = mrCluster.getConfig().get( + YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS); + String rmWebAppAddress1 = mrCluster.getConfig().get( + YarnConfiguration.RM_WEBAPP_ADDRESS); + + String mrHistAddress1 = mrCluster.getConfig().get( + JHAdminConfig.MR_HISTORY_ADDRESS); + String mrHistWebAppAddress1 = mrCluster.getConfig().get( + JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS); + + mrCluster.restart(); + + String rmAddress2 = mrCluster.getConfig().get(YarnConfiguration.RM_ADDRESS); + String rmAdminAddress2 = mrCluster.getConfig().get( + YarnConfiguration.RM_ADMIN_ADDRESS); + String rmSchedAddress2 = mrCluster.getConfig().get( + YarnConfiguration.RM_SCHEDULER_ADDRESS); + String rmRstrackerAddress2 = mrCluster.getConfig().get( + YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS); + String rmWebAppAddress2 = mrCluster.getConfig().get( + YarnConfiguration.RM_WEBAPP_ADDRESS); + + String mrHistAddress2 = mrCluster.getConfig().get( + JHAdminConfig.MR_HISTORY_ADDRESS); + String mrHistWebAppAddress2 = mrCluster.getConfig().get( + JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS); + + assertEquals("Address before restart: " + rmAddress1 + + " is different from new address: " + rmAddress2, rmAddress1, + rmAddress2); + assertEquals("Address before restart: " + rmAdminAddress1 + + " is different from new address: " + rmAdminAddress2, + rmAdminAddress1, rmAdminAddress2); + assertEquals("Address before restart: " + rmSchedAddress1 + + " is different from new address: " + rmSchedAddress2, + rmSchedAddress1, rmSchedAddress2); + assertEquals("Address before restart: " + rmRstrackerAddress1 + + " is different from new address: " + rmRstrackerAddress2, + rmRstrackerAddress1, rmRstrackerAddress2); + assertEquals("Address before restart: " + rmWebAppAddress1 + + " is different from new address: " + rmWebAppAddress2, + rmWebAppAddress1, rmWebAppAddress2); + assertEquals("Address before restart: " + mrHistAddress1 + + " is different from new address: " + mrHistAddress2, mrHistAddress1, + mrHistAddress2); + assertEquals("Address before restart: " + mrHistWebAppAddress1 + + " is different from new address: " + mrHistWebAppAddress2, + mrHistWebAppAddress1, mrHistWebAppAddress2); + + } + @Test public void testJob() throws Exception { final Job job = createJob();