From cb48bc1c93a4a1d8b2b936982a5e6b18494b5956 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Thu, 15 Sep 2011 10:53:05 +0000 Subject: [PATCH] MAPREDUCE-3007. Fixed Yarn Mapreduce client to be able to connect to JobHistoryServer in secure mode. Contributed by Vinod Kumar Vavilapalli. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1171051 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 117 +++++++++--------- .../org/apache/hadoop/mapred/ClientCache.java | 23 ++-- .../org.apache.hadoop.security.SecurityInfo | 1 + 3 files changed, 76 insertions(+), 65 deletions(-) create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index ac2f7c94ce..995aacc36d 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -1243,89 +1243,92 @@ Release 0.23.0 - Unreleased MAPREDUCE-2897. Javadoc for ClientRMProtocol protocol and related records. (acmurthy) - MAPREDUCE-2916. Ivy build for MRv1 fails with bad organization for - common daemon. (mahadev) + MAPREDUCE-2916. Ivy build for MRv1 fails with bad organization for + common daemon. (mahadev) - MAPREDUCE-2917. Fixed corner case in container reservation which led to - starvation and hung jobs. (acmurthy) + MAPREDUCE-2917. Fixed corner case in container reservation which led to + starvation and hung jobs. (acmurthy) - MAPREDUCE-2756. Better error handling in JobControl for failed jobs. - (Robert Evans via acmurthy) + MAPREDUCE-2756. Better error handling in JobControl for failed jobs. + (Robert Evans via acmurthy) - MAPREDUCE-2716. MRReliabilityTest job fails because of missing - job-file. (Jeffrey Naisbitt via vinodkv) + MAPREDUCE-2716. MRReliabilityTest job fails because of missing + job-file. (Jeffrey Naisbitt via vinodkv) - MAPREDUCE-2882. TestLineRecordReader depends on ant jars. (todd) + MAPREDUCE-2882. TestLineRecordReader depends on ant jars. (todd) - MAPREDUCE-2687. Fix NodeManager to use the right version of - LocalDirAllocator.getLocalPathToWrite. (mahadev & acmurthy) + MAPREDUCE-2687. Fix NodeManager to use the right version of + LocalDirAllocator.getLocalPathToWrite. (mahadev & acmurthy) - MAPREDUCE-2800. Set final progress for tasks to ensure all task information - is correctly logged to JobHistory. (Siddharth Seth via acmurthy) + MAPREDUCE-2800. Set final progress for tasks to ensure all task information + is correctly logged to JobHistory. (Siddharth Seth via acmurthy) - MAPREDUCE-2938. Log application submission failure in CapacityScheduler. - (acmurthy) + MAPREDUCE-2938. Log application submission failure in CapacityScheduler. + (acmurthy) - MAPREDUCE-2948. Hadoop streaming test failure, post MR-2767 (mahadev) + MAPREDUCE-2948. Hadoop streaming test failure, post MR-2767 (mahadev) - MAPREDUCE-2908. Fix all findbugs warnings. (vinodkv via acmurthy) + MAPREDUCE-2908. Fix all findbugs warnings. (vinodkv via acmurthy) - MAPREDUCE-2942. TestNMAuditLogger.testNMAuditLoggerWithIP failing (Thomas Graves - via mahadev) + MAPREDUCE-2942. TestNMAuditLogger.testNMAuditLoggerWithIP failing (Thomas Graves + via mahadev) - MAPREDUCE-2947. Fixed race condition in AuxiliaryServices. (vinodkv via - acmurthy) + MAPREDUCE-2947. Fixed race condition in AuxiliaryServices. (vinodkv via + acmurthy) - MAPREDUCE-2844. Fixed display of nodes in UI. (Ravi Teja Ch N V via - acmurthy) + MAPREDUCE-2844. Fixed display of nodes in UI. (Ravi Teja Ch N V via + acmurthy) - MAPREDUCE-2677. Fixed 404 for some links from HistoryServer. (Robert Evans - via acmurthy) + MAPREDUCE-2677. Fixed 404 for some links from HistoryServer. (Robert Evans + via acmurthy) - MAPREDUCE-2937. Ensure reason for application failure is displayed to the - user. (mahadev via acmurthy) + MAPREDUCE-2937. Ensure reason for application failure is displayed to the + user. (mahadev via acmurthy) - MAPREDUCE-2953. Fix a race condition on submission which caused client to - incorrectly assume application was gone by making submission synchronous - for RMAppManager. (Thomas Graves via acmurthy) + MAPREDUCE-2953. Fix a race condition on submission which caused client to + incorrectly assume application was gone by making submission synchronous + for RMAppManager. (Thomas Graves via acmurthy) - MAPREDUCE-2963. Fix hang in TestMRJobs. (Siddharth Seth via acmurthy) - - MAPREDUCE-2954. Fixed a deadlock in NM caused due to wrong synchronization - in protocol buffer records. (Siddharth Seth via vinodkv) - - MAPREDUCE-2975. Fixed YARNRunner to use YarnConfiguration rather than - Configuration. (mahadev via acmurthy) + MAPREDUCE-2963. Fix hang in TestMRJobs. (Siddharth Seth via acmurthy) - MAPREDUCE-2971. ant build mapreduce fails protected access jc.displayJobList - (jobs) (Thomas Graves via mahadev) + MAPREDUCE-2954. Fixed a deadlock in NM caused due to wrong synchronization + in protocol buffer records. (Siddharth Seth via vinodkv) - MAPREDUCE-2691. Finishing up the cleanup of distributed cache file resources - and related tests. (Siddharth Seth via vinodkv) + MAPREDUCE-2975. Fixed YARNRunner to use YarnConfiguration rather than + Configuration. (mahadev via acmurthy) + + MAPREDUCE-2971. ant build mapreduce fails protected access jc.displayJobList + (jobs) (Thomas Graves via mahadev) - MAPREDUCE-2749. Ensure NM registers with RM after starting all its services - correctly. (Thomas Graves via acmurthy) + MAPREDUCE-2691. Finishing up the cleanup of distributed cache file resources + and related tests. (Siddharth Seth via vinodkv) - MAPREDUCE-2979. Removed the needless ClientProtocolProvider configuration - from the hadoop-mapreduce-client-core module. (Siddharth Seth via vinodkv) + MAPREDUCE-2749. Ensure NM registers with RM after starting all its services + correctly. (Thomas Graves via acmurthy) - MAPREDUCE-2985. Fixed findbugs warnings in ResourceLocalizationService. - (Thomas Graves via acmurthy) + MAPREDUCE-2979. Removed the needless ClientProtocolProvider configuration + from the hadoop-mapreduce-client-core module. (Siddharth Seth via vinodkv) - MAPREDUCE-2874. Fix formatting of ApplicationId in web-ui. (Eric Payne via - acmurthy) + MAPREDUCE-2985. Fixed findbugs warnings in ResourceLocalizationService. + (Thomas Graves via acmurthy) - MAPREDUCE-2995. Better handling of expired containers in MapReduce - ApplicationMaster. (vinodkv via acmurthy) + MAPREDUCE-2874. Fix formatting of ApplicationId in web-ui. (Eric Payne via + acmurthy) - MAPREDUCE-2995. Fixed race condition in ContainerLauncher. (vinodkv via - acmurthy) + MAPREDUCE-2995. Better handling of expired containers in MapReduce + ApplicationMaster. (vinodkv via acmurthy) - MAPREDUCE-2949. Fixed NodeManager to shut-down correctly if a service - startup fails. (Ravi Teja via vinodkv) + MAPREDUCE-2995. Fixed race condition in ContainerLauncher. (vinodkv via + acmurthy) - MAPREDUCE-3005. Fix both FifoScheduler and CapacityScheduler to correctly - enforce locality constraints. (acmurthy) + MAPREDUCE-2949. Fixed NodeManager to shut-down correctly if a service + startup fails. (Ravi Teja via vinodkv) + + MAPREDUCE-3005. Fix both FifoScheduler and CapacityScheduler to correctly + enforce locality constraints. (acmurthy) + + MAPREDUCE-3007. Fixed Yarn Mapreduce client to be able to connect to + JobHistoryServer in secure mode. (vinodkv) Release 0.22.0 - Unreleased diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java index bc73b89256..50ffadeb84 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientCache.java @@ -19,6 +19,7 @@ package org.apache.hadoop.mapred; import java.io.IOException; +import java.security.PrivilegedAction; import java.util.HashMap; import java.util.Map; @@ -28,12 +29,13 @@ import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol; import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; +import org.apache.hadoop.mapreduce.v2.security.client.ClientHSSecurityInfo; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.SecurityInfo; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.ipc.YarnRPC; -import org.apache.hadoop.yarn.security.client.ClientRMSecurityInfo; public class ClientCache { @@ -72,16 +74,21 @@ synchronized ClientServiceDelegate getClient(JobID jobId) { private MRClientProtocol instantiateHistoryProxy() throws IOException { - String serviceAddr = conf.get(JHAdminConfig.MR_HISTORY_ADDRESS, + final String serviceAddr = conf.get(JHAdminConfig.MR_HISTORY_ADDRESS, JHAdminConfig.DEFAULT_MR_HISTORY_ADDRESS); LOG.info("Connecting to HistoryServer at: " + serviceAddr); - Configuration myConf = new Configuration(conf); - //TODO This should ideally be using it's own class (instead of ClientRMSecurityInfo) + final Configuration myConf = new Configuration(conf); myConf.setClass(YarnConfiguration.YARN_SECURITY_INFO, - ClientRMSecurityInfo.class, SecurityInfo.class); - YarnRPC rpc = YarnRPC.create(myConf); + ClientHSSecurityInfo.class, SecurityInfo.class); + final YarnRPC rpc = YarnRPC.create(myConf); LOG.info("Connected to HistoryServer at: " + serviceAddr); - return (MRClientProtocol) rpc.getProxy(MRClientProtocol.class, - NetUtils.createSocketAddr(serviceAddr), myConf); + UserGroupInformation currentUser = UserGroupInformation.getCurrentUser(); + return currentUser.doAs(new PrivilegedAction() { + @Override + public MRClientProtocol run() { + return (MRClientProtocol) rpc.getProxy(MRClientProtocol.class, + NetUtils.createSocketAddr(serviceAddr), myConf); + } + }); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo new file mode 100644 index 0000000000..10ab75be7d --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo @@ -0,0 +1 @@ +org.apache.hadoop.mapreduce.v2.security.client.ClientHSSecurityInfo