MAPREDUCE-3033. Ensure Master interface pays attention to classic v/s yarn frameworks. Contributed by Hitesh Shah.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1180214 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
54e81464a3
commit
6b343adfe8
@ -1546,6 +1546,9 @@ Release 0.23.0 - Unreleased
|
||||
MAPREDUCE-2751. Modified NodeManager to stop leaving around local files
|
||||
after application finishes. (Siddharth Seth via vinodkv)
|
||||
|
||||
MAPREDUCE-3033. Ensure Master interface pays attention to classic v/s yarn
|
||||
frameworks. (Hitesh Shah via acmurthy)
|
||||
|
||||
Release 0.22.0 - Unreleased
|
||||
|
||||
INCOMPATIBLE CHANGES
|
||||
|
@ -25,6 +25,7 @@
|
||||
import org.apache.hadoop.mapreduce.MRConfig;
|
||||
import org.apache.hadoop.net.NetUtils;
|
||||
import org.apache.hadoop.security.SecurityUtil;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
|
||||
public class Master {
|
||||
|
||||
@ -33,20 +34,33 @@ public enum State {
|
||||
}
|
||||
|
||||
public static String getMasterUserName(Configuration conf) {
|
||||
return conf.get(MRConfig.MASTER_USER_NAME);
|
||||
String framework = conf.get(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
|
||||
if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) {
|
||||
return conf.get(MRConfig.MASTER_USER_NAME);
|
||||
}
|
||||
else {
|
||||
return conf.get(YarnConfiguration.RM_PRINCIPAL);
|
||||
}
|
||||
}
|
||||
|
||||
public static InetSocketAddress getMasterAddress(Configuration conf) {
|
||||
String jobTrackerStr =
|
||||
conf.get(MRConfig.MASTER_ADDRESS, "localhost:8012");
|
||||
return NetUtils.createSocketAddr(jobTrackerStr);
|
||||
String masterAddress;
|
||||
String framework = conf.get(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
|
||||
if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) {
|
||||
masterAddress = conf.get(MRConfig.MASTER_ADDRESS, "localhost:8012");
|
||||
}
|
||||
else {
|
||||
masterAddress = conf.get(YarnConfiguration.RM_ADDRESS,
|
||||
YarnConfiguration.DEFAULT_RM_ADDRESS);
|
||||
}
|
||||
return NetUtils.createSocketAddr(masterAddress);
|
||||
}
|
||||
|
||||
public static String getMasterPrincipal(Configuration conf)
|
||||
throws IOException {
|
||||
String jtHostname = getMasterAddress(conf).getHostName();
|
||||
// get jobtracker principal for use as delegation token renewer
|
||||
return SecurityUtil.getServerPrincipal(getMasterUserName(conf), jtHostname);
|
||||
String masterHostname = getMasterAddress(conf).getHostName();
|
||||
// get kerberos principal for use as delegation token renewer
|
||||
return SecurityUtil.getServerPrincipal(getMasterUserName(conf), masterHostname);
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -343,7 +343,7 @@ public void run(JobConf job, final TaskUmbilicalProtocol umbilical)
|
||||
|
||||
boolean isLocal = false;
|
||||
// local iff framework == classic && master address == local
|
||||
String framework = job.get(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
|
||||
String framework = job.get(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
|
||||
if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) {
|
||||
isLocal = "local".equals(job.get(MRConfig.MASTER_ADDRESS, "local"));
|
||||
}
|
||||
|
@ -0,0 +1,88 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.mapred;
|
||||
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
import java.net.InetSocketAddress;
|
||||
|
||||
import org.apache.hadoop.mapreduce.MRConfig;
|
||||
import org.apache.hadoop.net.NetUtils;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.junit.Test;
|
||||
|
||||
public class TestMaster {
|
||||
|
||||
@Test
|
||||
public void testGetMasterAddress() {
|
||||
YarnConfiguration conf = new YarnConfiguration();
|
||||
|
||||
// Default is yarn framework
|
||||
String masterHostname = Master.getMasterAddress(conf).getHostName();
|
||||
|
||||
// no address set so should default to default rm address
|
||||
InetSocketAddress rmAddr = NetUtils.createSocketAddr(YarnConfiguration.DEFAULT_RM_ADDRESS);
|
||||
assertEquals(masterHostname, rmAddr.getHostName());
|
||||
|
||||
// Trying invalid master address for classic
|
||||
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
|
||||
conf.set(MRConfig.MASTER_ADDRESS, "local");
|
||||
|
||||
// should throw an exception for invalid value
|
||||
try {
|
||||
Master.getMasterAddress(conf);
|
||||
fail("Should not reach here as there is a bad master address");
|
||||
}
|
||||
catch (Exception e) {
|
||||
// Expected
|
||||
}
|
||||
|
||||
// Change master address to a valid value
|
||||
conf.set(MRConfig.MASTER_ADDRESS, "bar.com:9999");
|
||||
masterHostname = Master.getMasterAddress(conf).getHostName();
|
||||
assertEquals(masterHostname, "bar.com");
|
||||
|
||||
// change framework to yarn
|
||||
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
|
||||
conf.set(YarnConfiguration.RM_ADDRESS, "foo1.com:8192");
|
||||
masterHostname = Master.getMasterAddress(conf).getHostName();
|
||||
assertEquals(masterHostname, "foo1.com");
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetMasterUser() {
|
||||
YarnConfiguration conf = new YarnConfiguration();
|
||||
conf.set(MRConfig.MASTER_USER_NAME, "foo");
|
||||
conf.set(YarnConfiguration.RM_PRINCIPAL, "bar");
|
||||
|
||||
// default is yarn framework
|
||||
assertEquals(Master.getMasterUserName(conf), "bar");
|
||||
|
||||
// set framework name to classic
|
||||
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
|
||||
assertEquals(Master.getMasterUserName(conf), "foo");
|
||||
|
||||
// change framework to yarn
|
||||
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
|
||||
assertEquals(Master.getMasterUserName(conf), "bar");
|
||||
|
||||
}
|
||||
|
||||
}
|
@ -186,7 +186,7 @@ private ResourceMgrDelegate getRMDelegate() throws YarnRemoteException {
|
||||
private ClientServiceDelegate getClientServiceDelegate(
|
||||
MRClientProtocol historyServerProxy, ResourceMgrDelegate rm) {
|
||||
Configuration conf = new YarnConfiguration();
|
||||
conf.set(MRConfig.FRAMEWORK_NAME, "yarn");
|
||||
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
|
||||
ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate(
|
||||
conf, rm, oldJobId, historyServerProxy);
|
||||
return clientServiceDelegate;
|
||||
|
Loading…
Reference in New Issue
Block a user