From 6b343adfe880c7e9e2f441fd889cf16a8657c335 Mon Sep 17 00:00:00 2001 From: Arun Murthy Date: Fri, 7 Oct 2011 20:39:33 +0000 Subject: [PATCH] MAPREDUCE-3033. Ensure Master interface pays attention to classic v/s yarn frameworks. Contributed by Hitesh Shah. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1180214 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../java/org/apache/hadoop/mapred/Master.java | 28 ++++-- .../org/apache/hadoop/mapred/ReduceTask.java | 2 +- .../org/apache/hadoop/mapred/TestMaster.java | 88 +++++++++++++++++++ .../mapred/TestClientServiceDelegate.java | 2 +- 5 files changed, 114 insertions(+), 9 deletions(-) create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 90664a650e..dfe921af15 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -1546,6 +1546,9 @@ Release 0.23.0 - Unreleased MAPREDUCE-2751. Modified NodeManager to stop leaving around local files after application finishes. (Siddharth Seth via vinodkv) + MAPREDUCE-3033. Ensure Master interface pays attention to classic v/s yarn + frameworks. (Hitesh Shah via acmurthy) + Release 0.22.0 - Unreleased INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java index e2ab5fe3b9..6032ee02a6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java @@ -25,6 +25,7 @@ import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.yarn.conf.YarnConfiguration; public class Master { @@ -33,20 +34,33 @@ public enum State { } public static String getMasterUserName(Configuration conf) { - return conf.get(MRConfig.MASTER_USER_NAME); + String framework = conf.get(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); + if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) { + return conf.get(MRConfig.MASTER_USER_NAME); + } + else { + return conf.get(YarnConfiguration.RM_PRINCIPAL); + } } public static InetSocketAddress getMasterAddress(Configuration conf) { - String jobTrackerStr = - conf.get(MRConfig.MASTER_ADDRESS, "localhost:8012"); - return NetUtils.createSocketAddr(jobTrackerStr); + String masterAddress; + String framework = conf.get(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); + if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) { + masterAddress = conf.get(MRConfig.MASTER_ADDRESS, "localhost:8012"); + } + else { + masterAddress = conf.get(YarnConfiguration.RM_ADDRESS, + YarnConfiguration.DEFAULT_RM_ADDRESS); + } + return NetUtils.createSocketAddr(masterAddress); } public static String getMasterPrincipal(Configuration conf) throws IOException { - String jtHostname = getMasterAddress(conf).getHostName(); - // get jobtracker principal for use as delegation token renewer - return SecurityUtil.getServerPrincipal(getMasterUserName(conf), jtHostname); + String masterHostname = getMasterAddress(conf).getHostName(); + // get kerberos principal for use as delegation token renewer + return SecurityUtil.getServerPrincipal(getMasterUserName(conf), masterHostname); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java index f278c8528c..dafa6ce7dc 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java @@ -343,7 +343,7 @@ public void run(JobConf job, final TaskUmbilicalProtocol umbilical) boolean isLocal = false; // local iff framework == classic && master address == local - String framework = job.get(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME); + String framework = job.get(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) { isLocal = "local".equals(job.get(MRConfig.MASTER_ADDRESS, "local")); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java new file mode 100644 index 0000000000..132af0ba2b --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java @@ -0,0 +1,88 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.mapred; + +import static org.junit.Assert.*; + +import java.net.InetSocketAddress; + +import org.apache.hadoop.mapreduce.MRConfig; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.junit.Test; + +public class TestMaster { + + @Test + public void testGetMasterAddress() { + YarnConfiguration conf = new YarnConfiguration(); + + // Default is yarn framework + String masterHostname = Master.getMasterAddress(conf).getHostName(); + + // no address set so should default to default rm address + InetSocketAddress rmAddr = NetUtils.createSocketAddr(YarnConfiguration.DEFAULT_RM_ADDRESS); + assertEquals(masterHostname, rmAddr.getHostName()); + + // Trying invalid master address for classic + conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME); + conf.set(MRConfig.MASTER_ADDRESS, "local"); + + // should throw an exception for invalid value + try { + Master.getMasterAddress(conf); + fail("Should not reach here as there is a bad master address"); + } + catch (Exception e) { + // Expected + } + + // Change master address to a valid value + conf.set(MRConfig.MASTER_ADDRESS, "bar.com:9999"); + masterHostname = Master.getMasterAddress(conf).getHostName(); + assertEquals(masterHostname, "bar.com"); + + // change framework to yarn + conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); + conf.set(YarnConfiguration.RM_ADDRESS, "foo1.com:8192"); + masterHostname = Master.getMasterAddress(conf).getHostName(); + assertEquals(masterHostname, "foo1.com"); + + } + + @Test + public void testGetMasterUser() { + YarnConfiguration conf = new YarnConfiguration(); + conf.set(MRConfig.MASTER_USER_NAME, "foo"); + conf.set(YarnConfiguration.RM_PRINCIPAL, "bar"); + + // default is yarn framework + assertEquals(Master.getMasterUserName(conf), "bar"); + + // set framework name to classic + conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME); + assertEquals(Master.getMasterUserName(conf), "foo"); + + // change framework to yarn + conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); + assertEquals(Master.getMasterUserName(conf), "bar"); + + } + +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java index d04d7e3d99..f21877c4b4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java @@ -186,7 +186,7 @@ private ResourceMgrDelegate getRMDelegate() throws YarnRemoteException { private ClientServiceDelegate getClientServiceDelegate( MRClientProtocol historyServerProxy, ResourceMgrDelegate rm) { Configuration conf = new YarnConfiguration(); - conf.set(MRConfig.FRAMEWORK_NAME, "yarn"); + conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate( conf, rm, oldJobId, historyServerProxy); return clientServiceDelegate;