diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml index 4de24c78a8..00bb11bd77 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml @@ -37,7 +37,11 @@ org.apache.hadoop - hadoop-yarn-common + hadoop-yarn-client + + + org.apache.hadoop + hadoop-yarn-common org.apache.hadoop diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java index d84e395019..ec004050e2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java @@ -19,75 +19,45 @@ package org.apache.hadoop.mapred; import java.io.IOException; -import java.net.InetSocketAddress; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.SecurityUtil; -import org.apache.hadoop.yarn.conf.HAUtil; -import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.client.util.YarnClientUtils; @Private @Unstable public class Master { - - private static final Log LOG = LogFactory.getLog(Master.class); - public enum State { INITIALIZING, RUNNING; } - public static String getMasterUserName(Configuration conf) { - String framework = conf.get(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); - if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) { - return conf.get(MRConfig.MASTER_USER_NAME); - } - else { - return conf.get(YarnConfiguration.RM_PRINCIPAL); - } - } - - public static InetSocketAddress getMasterAddress(Configuration conf) { - String masterAddress; - String framework = conf.get(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); - if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) { - masterAddress = conf.get(MRConfig.MASTER_ADDRESS, "localhost:8012"); - return NetUtils.createSocketAddr(masterAddress, 8012, MRConfig.MASTER_ADDRESS); - } else if (framework.equals(MRConfig.YARN_FRAMEWORK_NAME) && - HAUtil.isHAEnabled(conf)) { - YarnConfiguration yarnConf = new YarnConfiguration(conf); - if (yarnConf.get(YarnConfiguration.RM_HA_ID) == null) { - String[] rmIds = yarnConf.getStrings(YarnConfiguration.RM_HA_IDS); - if (rmIds != null && rmIds.length > 0) { - // If RM_HA_ID is not configured, use the first one. - // Because any valid RM HA ID should work. - yarnConf.set(YarnConfiguration.RM_HA_ID, rmIds[0]); - } else { - LOG.warn("RM_HA_IDS is not configured when RM HA is enabled"); - } - } - return yarnConf.getSocketAddr( - YarnConfiguration.RM_ADDRESS, - YarnConfiguration.DEFAULT_RM_ADDRESS, - YarnConfiguration.DEFAULT_RM_PORT); - } else { - return conf.getSocketAddr( - YarnConfiguration.RM_ADDRESS, - YarnConfiguration.DEFAULT_RM_ADDRESS, - YarnConfiguration.DEFAULT_RM_PORT); - } + public static String getMasterAddress(Configuration conf) { + String masterAddress = conf.get(MRConfig.MASTER_ADDRESS, "localhost:8012"); + + return NetUtils.createSocketAddr(masterAddress, 8012, + MRConfig.MASTER_ADDRESS).getHostName(); } - public static String getMasterPrincipal(Configuration conf) - throws IOException { - String masterHostname = getMasterAddress(conf).getHostName(); - // get kerberos principal for use as delegation token renewer - return SecurityUtil.getServerPrincipal(getMasterUserName(conf), masterHostname); + public static String getMasterPrincipal(Configuration conf) + throws IOException { + String masterPrincipal; + String framework = conf.get(MRConfig.FRAMEWORK_NAME, + MRConfig.YARN_FRAMEWORK_NAME); + + if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) { + String masterAddress = getMasterAddress(conf); + // get kerberos principal for use as delegation token renewer + masterPrincipal = + SecurityUtil.getServerPrincipal(conf.get(MRConfig.MASTER_USER_NAME), + masterAddress); + } else { + masterPrincipal = YarnClientUtils.getRmPrincipal(conf); + } + + return masterPrincipal; } - } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java index 498abbc0ee..0f4ebeef89 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java @@ -18,13 +18,10 @@ package org.apache.hadoop.mapred; -import static org.junit.Assert.*; - -import java.net.InetSocketAddress; - import org.apache.hadoop.mapreduce.MRConfig; -import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; import org.junit.Test; public class TestMaster { @@ -33,13 +30,6 @@ public class TestMaster { public void testGetMasterAddress() { YarnConfiguration conf = new YarnConfiguration(); - // Default is yarn framework - String masterHostname = Master.getMasterAddress(conf).getHostName(); - - // no address set so should default to default rm address - InetSocketAddress rmAddr = NetUtils.createSocketAddr(YarnConfiguration.DEFAULT_RM_ADDRESS); - assertEquals(masterHostname, rmAddr.getHostName()); - // Trying invalid master address for classic conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME); conf.set(MRConfig.MASTER_ADDRESS, "local:invalid"); @@ -55,47 +45,7 @@ public void testGetMasterAddress() { // Change master address to a valid value conf.set(MRConfig.MASTER_ADDRESS, "bar.com:8042"); - masterHostname = Master.getMasterAddress(conf).getHostName(); + String masterHostname = Master.getMasterAddress(conf); assertEquals(masterHostname, "bar.com"); - - // change framework to yarn - conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); - conf.set(YarnConfiguration.RM_ADDRESS, "foo1.com:8192"); - masterHostname = Master.getMasterAddress(conf).getHostName(); - assertEquals(masterHostname, "foo1.com"); - - // change framework to yarn and enable HA - conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); - conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); - conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2"); - conf.set(YarnConfiguration.RM_ADDRESS + ".rm1", "rm1.com:8192"); - conf.set(YarnConfiguration.RM_ADDRESS + ".rm2", "rm2.com:8192"); - masterHostname = Master.getMasterAddress(conf).getHostName(); - // If RM_HA_ID is not configured, the first one in RM_HA_IDS will be used. - assertEquals(masterHostname, "rm1.com"); - conf.set(YarnConfiguration.RM_HA_ID, "rm2"); - masterHostname = Master.getMasterAddress(conf).getHostName(); - // If RM_HA_ID is configured, use the given RM_HA_ID. - assertEquals(masterHostname, "rm2.com"); } - - @Test - public void testGetMasterUser() { - YarnConfiguration conf = new YarnConfiguration(); - conf.set(MRConfig.MASTER_USER_NAME, "foo"); - conf.set(YarnConfiguration.RM_PRINCIPAL, "bar"); - - // default is yarn framework - assertEquals(Master.getMasterUserName(conf), "bar"); - - // set framework name to classic - conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME); - assertEquals(Master.getMasterUserName(conf), "foo"); - - // change framework to yarn - conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); - assertEquals(Master.getMasterUserName(conf), "bar"); - - } - }