MAPREDUCE-6484. Yarn Client uses local address instead of RM address as token renewer in a secure cluster when RM HA is enabled. Contributed by Zhihai Xu

This commit is contained in:
Zhihai Xu 2015-09-24 08:12:07 -07:00
parent e52bc697f8
commit 97a08807ec
3 changed files with 40 additions and 3 deletions

View File

@ -577,6 +577,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6460. TestRMContainerAllocator. MAPREDUCE-6460. TestRMContainerAllocator.
testAttemptNotFoundCausesRMCommunicatorException fails. (Zhihai Xu) testAttemptNotFoundCausesRMCommunicatorException fails. (Zhihai Xu)
MAPREDUCE-6484. Yarn Client uses local address instead of RM address as
token renewer in a secure cluster when RM HA is enabled. (Zhihai Xu)
Release 2.7.2 - UNRELEASED Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

View File

@ -21,18 +21,23 @@
import java.io.IOException; import java.io.IOException;
import java.net.InetSocketAddress; import java.net.InetSocketAddress;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
@Private @Private
@Unstable @Unstable
public class Master { public class Master {
private static final Log LOG = LogFactory.getLog(Master.class);
public enum State { public enum State {
INITIALIZING, RUNNING; INITIALIZING, RUNNING;
} }
@ -53,8 +58,24 @@ public static InetSocketAddress getMasterAddress(Configuration conf) {
if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) { if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) {
masterAddress = conf.get(MRConfig.MASTER_ADDRESS, "localhost:8012"); masterAddress = conf.get(MRConfig.MASTER_ADDRESS, "localhost:8012");
return NetUtils.createSocketAddr(masterAddress, 8012, MRConfig.MASTER_ADDRESS); return NetUtils.createSocketAddr(masterAddress, 8012, MRConfig.MASTER_ADDRESS);
} else if (framework.equals(MRConfig.YARN_FRAMEWORK_NAME) &&
HAUtil.isHAEnabled(conf)) {
YarnConfiguration yarnConf = new YarnConfiguration(conf);
if (yarnConf.get(YarnConfiguration.RM_HA_ID) == null) {
String[] rmIds = yarnConf.getStrings(YarnConfiguration.RM_HA_IDS);
if (rmIds != null && rmIds.length > 0) {
// If RM_HA_ID is not configured, use the first one.
// Because any valid RM HA ID should work.
yarnConf.set(YarnConfiguration.RM_HA_ID, rmIds[0]);
} else {
LOG.warn("RM_HA_IDS is not configured when RM HA is enabled");
} }
else { }
return yarnConf.getSocketAddr(
YarnConfiguration.RM_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADDRESS,
YarnConfiguration.DEFAULT_RM_PORT);
} else {
return conf.getSocketAddr( return conf.getSocketAddr(
YarnConfiguration.RM_ADDRESS, YarnConfiguration.RM_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS,

View File

@ -64,6 +64,19 @@ public void testGetMasterAddress() {
masterHostname = Master.getMasterAddress(conf).getHostName(); masterHostname = Master.getMasterAddress(conf).getHostName();
assertEquals(masterHostname, "foo1.com"); assertEquals(masterHostname, "foo1.com");
// change framework to yarn and enable HA
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2");
conf.set(YarnConfiguration.RM_ADDRESS + ".rm1", "rm1.com:8192");
conf.set(YarnConfiguration.RM_ADDRESS + ".rm2", "rm2.com:8192");
masterHostname = Master.getMasterAddress(conf).getHostName();
// If RM_HA_ID is not configured, the first one in RM_HA_IDS will be used.
assertEquals(masterHostname, "rm1.com");
conf.set(YarnConfiguration.RM_HA_ID, "rm2");
masterHostname = Master.getMasterAddress(conf).getHostName();
// If RM_HA_ID is configured, use the given RM_HA_ID.
assertEquals(masterHostname, "rm2.com");
} }
@Test @Test