big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-6632. Master.getMasterAddress() should be updated to use YARN-4629 (templedf via rkanter)

Browse Source
This commit is contained in:
Robert Kanter 2016-09-22 16:12:56 -07:00
parent 40acacee08
commit 4fc632ae19
3 changed files with 31 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
View File

@ -37,7 +37,11 @@
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
<artifactId>hadoop-yarn-client</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>

76
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java
View File

@ -19,75 +19,45 @@
package org.apache.hadoop.mapred;
import java.io.IOException;
import java.net.InetSocketAddress;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.client.util.YarnClientUtils;
@Private
@Unstable
public class Master {
private static final Log LOG = LogFactory.getLog(Master.class);
public enum State {
INITIALIZING, RUNNING;
}
public static String getMasterUserName(Configuration conf) {
String framework = conf.get(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) {
return conf.get(MRConfig.MASTER_USER_NAME);
}
else {
return conf.get(YarnConfiguration.RM_PRINCIPAL);
}
}
public static InetSocketAddress getMasterAddress(Configuration conf) {
String masterAddress;
String framework = conf.get(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) {
masterAddress = conf.get(MRConfig.MASTER_ADDRESS, "localhost:8012");
return NetUtils.createSocketAddr(masterAddress, 8012, MRConfig.MASTER_ADDRESS);
} else if (framework.equals(MRConfig.YARN_FRAMEWORK_NAME) &&
HAUtil.isHAEnabled(conf)) {
YarnConfiguration yarnConf = new YarnConfiguration(conf);
if (yarnConf.get(YarnConfiguration.RM_HA_ID) == null) {
String[] rmIds = yarnConf.getStrings(YarnConfiguration.RM_HA_IDS);
if (rmIds != null && rmIds.length > 0) {
// If RM_HA_ID is not configured, use the first one.
// Because any valid RM HA ID should work.
yarnConf.set(YarnConfiguration.RM_HA_ID, rmIds[0]);
} else {
LOG.warn("RM_HA_IDS is not configured when RM HA is enabled");
}
}
return yarnConf.getSocketAddr(
YarnConfiguration.RM_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADDRESS,
YarnConfiguration.DEFAULT_RM_PORT);
} else {
return conf.getSocketAddr(
YarnConfiguration.RM_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADDRESS,
YarnConfiguration.DEFAULT_RM_PORT);
}
public static String getMasterAddress(Configuration conf) {
String masterAddress = conf.get(MRConfig.MASTER_ADDRESS, "localhost:8012");
return NetUtils.createSocketAddr(masterAddress, 8012,
MRConfig.MASTER_ADDRESS).getHostName();
}
public static String getMasterPrincipal(Configuration conf)
throws IOException {
String masterHostname = getMasterAddress(conf).getHostName();
// get kerberos principal for use as delegation token renewer
return SecurityUtil.getServerPrincipal(getMasterUserName(conf), masterHostname);
public static String getMasterPrincipal(Configuration conf)
throws IOException {
String masterPrincipal;
String framework = conf.get(MRConfig.FRAMEWORK_NAME,
MRConfig.YARN_FRAMEWORK_NAME);
if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) {
String masterAddress = getMasterAddress(conf);
// get kerberos principal for use as delegation token renewer
masterPrincipal =
SecurityUtil.getServerPrincipal(conf.get(MRConfig.MASTER_USER_NAME),
masterAddress);
} else {
masterPrincipal = YarnClientUtils.getRmPrincipal(conf);
}
return masterPrincipal;
}
}

56
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java
View File

@ -18,13 +18,10 @@
package org.apache.hadoop.mapred;
import static org.junit.Assert.*;
import java.net.InetSocketAddress;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import org.junit.Test;
public class TestMaster {
@ -33,13 +30,6 @@ public class TestMaster {
public void testGetMasterAddress() {
YarnConfiguration conf = new YarnConfiguration();
// Default is yarn framework
String masterHostname = Master.getMasterAddress(conf).getHostName();
// no address set so should default to default rm address
InetSocketAddress rmAddr = NetUtils.createSocketAddr(YarnConfiguration.DEFAULT_RM_ADDRESS);
assertEquals(masterHostname, rmAddr.getHostName());
// Trying invalid master address for classic
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
conf.set(MRConfig.MASTER_ADDRESS, "local:invalid");
@ -55,47 +45,7 @@ public void testGetMasterAddress() {
// Change master address to a valid value
conf.set(MRConfig.MASTER_ADDRESS, "bar.com:8042");
masterHostname = Master.getMasterAddress(conf).getHostName();
String masterHostname = Master.getMasterAddress(conf);
assertEquals(masterHostname, "bar.com");
// change framework to yarn
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
conf.set(YarnConfiguration.RM_ADDRESS, "foo1.com:8192");
masterHostname = Master.getMasterAddress(conf).getHostName();
assertEquals(masterHostname, "foo1.com");
// change framework to yarn and enable HA
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2");
conf.set(YarnConfiguration.RM_ADDRESS + ".rm1", "rm1.com:8192");
conf.set(YarnConfiguration.RM_ADDRESS + ".rm2", "rm2.com:8192");
masterHostname = Master.getMasterAddress(conf).getHostName();
// If RM_HA_ID is not configured, the first one in RM_HA_IDS will be used.
assertEquals(masterHostname, "rm1.com");
conf.set(YarnConfiguration.RM_HA_ID, "rm2");
masterHostname = Master.getMasterAddress(conf).getHostName();
// If RM_HA_ID is configured, use the given RM_HA_ID.
assertEquals(masterHostname, "rm2.com");
}
@Test
public void testGetMasterUser() {
YarnConfiguration conf = new YarnConfiguration();
conf.set(MRConfig.MASTER_USER_NAME, "foo");
conf.set(YarnConfiguration.RM_PRINCIPAL, "bar");
// default is yarn framework
assertEquals(Master.getMasterUserName(conf), "bar");
// set framework name to classic
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
assertEquals(Master.getMasterUserName(conf), "foo");
// change framework to yarn
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
assertEquals(Master.getMasterUserName(conf), "bar");
}
}