From 09b3dee122ad20fae22ae8de74886dbf11c39343 Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Mon, 17 Nov 2014 22:59:59 +0000 Subject: [PATCH] MAPREDUCE-6162. mapred hsadmin fails on a secure cluster. Contributed by Jason Lowe --- hadoop-mapreduce-project/CHANGES.txt | 6 ++-- .../authorize/ClientHSPolicyProvider.java | 17 +++++++++-- .../hadoop-mapreduce-client-common/pom.xml | 1 + .../mapreduce/v2/api}/HSAdminProtocol.java | 6 +--- .../v2/api}/HSAdminRefreshProtocol.java | 6 +--- .../v2/api}/HSAdminRefreshProtocolPB.java | 4 +-- .../v2/jobhistory/JHAdminConfig.java | 4 ++- .../main/proto/HSAdminRefreshProtocol.proto | 0 .../hadoop-mapreduce-client-hs/pom.xml | 30 ------------------- .../hadoop/mapreduce/v2/hs/HSProxies.java | 4 +-- .../mapreduce/v2/hs/client/HSAdmin.java | 18 ++++++++++- ...RefreshProtocolClientSideTranslatorPB.java | 3 +- ...RefreshProtocolServerSideTranslatorPB.java | 3 +- .../mapreduce/v2/hs/server/HSAdminServer.java | 14 +++++++-- .../v2/hs/server/TestHSAdminServer.java | 20 +++++++++++++ 15 files changed, 81 insertions(+), 55 deletions(-) rename hadoop-mapreduce-project/hadoop-mapreduce-client/{hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocol => hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api}/HSAdminProtocol.java (94%) rename hadoop-mapreduce-project/hadoop-mapreduce-client/{hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocol => hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api}/HSAdminRefreshProtocol.java (95%) rename hadoop-mapreduce-project/hadoop-mapreduce-client/{hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocolPB => hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api}/HSAdminRefreshProtocolPB.java (89%) rename hadoop-mapreduce-project/hadoop-mapreduce-client/{hadoop-mapreduce-client-hs => hadoop-mapreduce-client-common}/src/main/proto/HSAdminRefreshProtocol.proto (100%) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 653ad44f81..cd5f965c81 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -239,9 +239,11 @@ Release 2.7.0 - UNRELEASED BUG FIXES - MAPREDUCE-5918. LineRecordReader can return the same decompressor to CodecPool - multiple times (Sergey Murylev via raviprak) + MAPREDUCE-5918. LineRecordReader can return the same decompressor to + CodecPool multiple times (Sergey Murylev via raviprak) + MAPREDUCE-6162. mapred hsadmin fails on a secure cluster (jlowe) + Release 2.6.0 - 2014-11-18 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/security/authorize/ClientHSPolicyProvider.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/security/authorize/ClientHSPolicyProvider.java index dfb7469490..a3bf8aefa2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/security/authorize/ClientHSPolicyProvider.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/security/authorize/ClientHSPolicyProvider.java @@ -19,13 +19,17 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.mapreduce.v2.api.HSAdminRefreshProtocol; import org.apache.hadoop.mapreduce.v2.api.HSClientProtocolPB; import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; +import org.apache.hadoop.security.RefreshUserMappingsProtocol; import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.security.authorize.Service; +import org.apache.hadoop.tools.GetUserMappingsProtocol; /** - * {@link PolicyProvider} for YARN MapReduce protocols. + * {@link PolicyProvider} for MapReduce history server protocols. */ @InterfaceAudience.Private @InterfaceStability.Unstable @@ -35,7 +39,16 @@ public class ClientHSPolicyProvider extends PolicyProvider { new Service[] { new Service( JHAdminConfig.MR_HS_SECURITY_SERVICE_AUTHORIZATION, - HSClientProtocolPB.class) + HSClientProtocolPB.class), + new Service( + CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_AUTHORIZATION_GET_USER_MAPPINGS, + GetUserMappingsProtocol.class), + new Service( + CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_AUTHORIZATION_REFRESH_USER_MAPPINGS, + RefreshUserMappingsProtocol.class), + new Service( + JHAdminConfig.MR_HS_SECURITY_SERVICE_AUTHORIZATION_ADMIN_REFRESH, + HSAdminRefreshProtocol.class) }; @Override diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml index 7ef7d3b038..ab00baa36e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml @@ -74,6 +74,7 @@ ${basedir}/src/main/proto + HSAdminRefreshProtocol.proto mr_protos.proto mr_service_protos.proto MRClientProtocol.proto diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocol/HSAdminProtocol.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/HSAdminProtocol.java similarity index 94% rename from hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocol/HSAdminProtocol.java rename to hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/HSAdminProtocol.java index ec75aa7b12..c06904093f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocol/HSAdminProtocol.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/HSAdminProtocol.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.mapreduce.v2.hs.protocol; +package org.apache.hadoop.mapreduce.v2.api; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability; @@ -25,10 +25,6 @@ import org.apache.hadoop.security.RefreshUserMappingsProtocol; import org.apache.hadoop.tools.GetUserMappingsProtocol; -/** - * Protocol use - * - */ @KerberosInfo(serverPrincipal = CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY) @Private @InterfaceStability.Evolving diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocol/HSAdminRefreshProtocol.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/HSAdminRefreshProtocol.java similarity index 95% rename from hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocol/HSAdminRefreshProtocol.java rename to hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/HSAdminRefreshProtocol.java index 82d9ae2d51..d1b8714817 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocol/HSAdminRefreshProtocol.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/HSAdminRefreshProtocol.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.mapreduce.v2.hs.protocol; +package org.apache.hadoop.mapreduce.v2.api; import java.io.IOException; @@ -25,10 +25,6 @@ import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.security.KerberosInfo; -/** - * Protocol use - * - */ @KerberosInfo(serverPrincipal = CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY) @Private @InterfaceStability.Evolving diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocolPB/HSAdminRefreshProtocolPB.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/HSAdminRefreshProtocolPB.java similarity index 89% rename from hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocolPB/HSAdminRefreshProtocolPB.java rename to hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/HSAdminRefreshProtocolPB.java index 1831dd8aa1..93fe752ea5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocolPB/HSAdminRefreshProtocolPB.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/HSAdminRefreshProtocolPB.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.mapreduce.v2.hs.protocolPB; +package org.apache.hadoop.mapreduce.v2.api; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability; @@ -25,7 +25,7 @@ import org.apache.hadoop.security.KerberosInfo; @KerberosInfo(serverPrincipal = CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY) -@ProtocolInfo(protocolName = "org.apache.hadoop.mapreduce.v2.hs.protocol.HSAdminRefreshProtocol", protocolVersion = 1) +@ProtocolInfo(protocolName = "org.apache.hadoop.mapreduce.v2.api.HSAdminRefreshProtocol", protocolVersion = 1) @Private @InterfaceStability.Evolving public interface HSAdminRefreshProtocolPB extends diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java index 9fa8a090a4..e5a49b5e1b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java @@ -167,7 +167,9 @@ public class JHAdminConfig { */ public static final String MR_HS_SECURITY_SERVICE_AUTHORIZATION = "security.mrhs.client.protocol.acl"; - + public static final String MR_HS_SECURITY_SERVICE_AUTHORIZATION_ADMIN_REFRESH = + "security.mrhs.admin.refresh.protocol.acl"; + /** * The HistoryStorage class to use to cache history data. */ diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/proto/HSAdminRefreshProtocol.proto b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/HSAdminRefreshProtocol.proto similarity index 100% rename from hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/proto/HSAdminRefreshProtocol.proto rename to hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/HSAdminRefreshProtocol.proto diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml index ad887bba63..adeb9fa621 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml @@ -77,36 +77,6 @@ - - org.apache.hadoop - hadoop-maven-plugins - - - compile-protoc - generate-sources - - protoc - - - ${protobuf.version} - ${protoc.path} - - - ${basedir}/../../../../hadoop-common-project/hadoop-common/src/main/proto - - ${basedir}/src/main/proto - - - ${basedir}/src/main/proto - - HSAdminRefreshProtocol.proto - - - ${project.build.directory}/generated-sources/java - - - - diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HSProxies.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HSProxies.java index 64445f6c52..65fe950a70 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HSProxies.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HSProxies.java @@ -27,9 +27,9 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ipc.ProtobufRpcEngine; import org.apache.hadoop.ipc.RPC; -import org.apache.hadoop.mapreduce.v2.hs.protocol.HSAdminRefreshProtocol; +import org.apache.hadoop.mapreduce.v2.api.HSAdminRefreshProtocol; +import org.apache.hadoop.mapreduce.v2.api.HSAdminRefreshProtocolPB; import org.apache.hadoop.mapreduce.v2.hs.protocolPB.HSAdminRefreshProtocolClientSideTranslatorPB; -import org.apache.hadoop.mapreduce.v2.hs.protocolPB.HSAdminRefreshProtocolPB; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.RefreshUserMappingsProtocol; import org.apache.hadoop.security.UserGroupInformation; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/client/HSAdmin.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/client/HSAdmin.java index 000ea54618..e0d3a68e05 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/client/HSAdmin.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/client/HSAdmin.java @@ -25,9 +25,10 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapreduce.v2.api.HSAdminRefreshProtocol; import org.apache.hadoop.mapreduce.v2.hs.HSProxies; -import org.apache.hadoop.mapreduce.v2.hs.protocol.HSAdminRefreshProtocol; import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; import org.apache.hadoop.security.RefreshUserMappingsProtocol; import org.apache.hadoop.security.UserGroupInformation; @@ -46,6 +47,21 @@ public HSAdmin(JobConf conf) { super(conf); } + @Override + public void setConf(Configuration conf) { + if (conf != null) { + conf = addSecurityConfiguration(conf); + } + super.setConf(conf); + } + + private Configuration addSecurityConfiguration(Configuration conf) { + conf = new JobConf(conf); + conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY, + conf.get(JHAdminConfig.MR_HISTORY_PRINCIPAL, "")); + return conf; + } + /** * Displays format of commands. * diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocolPB/HSAdminRefreshProtocolClientSideTranslatorPB.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocolPB/HSAdminRefreshProtocolClientSideTranslatorPB.java index 8f1f512676..e014f62f3f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocolPB/HSAdminRefreshProtocolClientSideTranslatorPB.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocolPB/HSAdminRefreshProtocolClientSideTranslatorPB.java @@ -26,11 +26,12 @@ import org.apache.hadoop.ipc.ProtocolMetaInterface; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.RpcClientUtil; +import org.apache.hadoop.mapreduce.v2.api.HSAdminRefreshProtocol; +import org.apache.hadoop.mapreduce.v2.api.HSAdminRefreshProtocolPB; import org.apache.hadoop.mapreduce.v2.hs.proto.HSAdminRefreshProtocolProtos.RefreshAdminAclsRequestProto; import org.apache.hadoop.mapreduce.v2.hs.proto.HSAdminRefreshProtocolProtos.RefreshLoadedJobCacheRequestProto; import org.apache.hadoop.mapreduce.v2.hs.proto.HSAdminRefreshProtocolProtos.RefreshJobRetentionSettingsRequestProto; import org.apache.hadoop.mapreduce.v2.hs.proto.HSAdminRefreshProtocolProtos.RefreshLogRetentionSettingsRequestProto; -import org.apache.hadoop.mapreduce.v2.hs.protocol.HSAdminRefreshProtocol; import com.google.protobuf.RpcController; import com.google.protobuf.ServiceException; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocolPB/HSAdminRefreshProtocolServerSideTranslatorPB.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocolPB/HSAdminRefreshProtocolServerSideTranslatorPB.java index 77b4178e04..6b51889180 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocolPB/HSAdminRefreshProtocolServerSideTranslatorPB.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/protocolPB/HSAdminRefreshProtocolServerSideTranslatorPB.java @@ -21,6 +21,8 @@ import java.io.IOException; import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.mapreduce.v2.api.HSAdminRefreshProtocol; +import org.apache.hadoop.mapreduce.v2.api.HSAdminRefreshProtocolPB; import org.apache.hadoop.mapreduce.v2.hs.proto.HSAdminRefreshProtocolProtos.RefreshAdminAclsResponseProto; import org.apache.hadoop.mapreduce.v2.hs.proto.HSAdminRefreshProtocolProtos.RefreshAdminAclsRequestProto; import org.apache.hadoop.mapreduce.v2.hs.proto.HSAdminRefreshProtocolProtos.RefreshLoadedJobCacheRequestProto; @@ -29,7 +31,6 @@ import org.apache.hadoop.mapreduce.v2.hs.proto.HSAdminRefreshProtocolProtos.RefreshJobRetentionSettingsResponseProto; import org.apache.hadoop.mapreduce.v2.hs.proto.HSAdminRefreshProtocolProtos.RefreshLogRetentionSettingsRequestProto; import org.apache.hadoop.mapreduce.v2.hs.proto.HSAdminRefreshProtocolProtos.RefreshLogRetentionSettingsResponseProto; -import org.apache.hadoop.mapreduce.v2.hs.protocol.HSAdminRefreshProtocol; import com.google.protobuf.RpcController; import com.google.protobuf.ServiceException; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java index 858d945dfe..ef79bf87f6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java @@ -25,6 +25,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.ipc.ProtobufRpcEngine; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.WritableRpcEngine; @@ -34,7 +35,6 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.authorize.ProxyUsers; -import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.logaggregation.AggregatedLogDeletionService; import org.apache.hadoop.security.proto.RefreshUserMappingsProtocolProtos.RefreshUserMappingsProtocolService; import org.apache.hadoop.security.protocolPB.RefreshUserMappingsProtocolPB; @@ -43,12 +43,13 @@ import org.apache.hadoop.tools.proto.GetUserMappingsProtocolProtos.GetUserMappingsProtocolService; import org.apache.hadoop.tools.protocolPB.GetUserMappingsProtocolPB; import org.apache.hadoop.tools.protocolPB.GetUserMappingsProtocolServerSideTranslatorPB; +import org.apache.hadoop.mapreduce.v2.api.HSAdminProtocol; +import org.apache.hadoop.mapreduce.v2.api.HSAdminRefreshProtocolPB; +import org.apache.hadoop.mapreduce.v2.app.security.authorize.ClientHSPolicyProvider; import org.apache.hadoop.mapreduce.v2.hs.HSAuditLogger; import org.apache.hadoop.mapreduce.v2.hs.HSAuditLogger.AuditConstants; import org.apache.hadoop.mapreduce.v2.hs.JobHistory; import org.apache.hadoop.mapreduce.v2.hs.proto.HSAdminRefreshProtocolProtos.HSAdminRefreshProtocolService; -import org.apache.hadoop.mapreduce.v2.hs.protocol.HSAdminProtocol; -import org.apache.hadoop.mapreduce.v2.hs.protocolPB.HSAdminRefreshProtocolPB; import org.apache.hadoop.mapreduce.v2.hs.protocolPB.HSAdminRefreshProtocolServerSideTranslatorPB; import com.google.protobuf.BlockingService; @@ -110,6 +111,13 @@ public void serviceInit(Configuration conf) throws Exception { addProtocol(conf, HSAdminRefreshProtocolPB.class, refreshHSAdminProtocolService); + // Enable service authorization? + if (conf.getBoolean( + CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, + false)) { + clientRpcServer.refreshServiceAcl(conf, new ClientHSPolicyProvider()); + } + adminAcl = new AccessControlList(conf.get(JHAdminConfig.JHS_ADMIN_ACL, JHAdminConfig.DEFAULT_JHS_ADMIN_ACL)); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java index 2c239ec2bb..7ab90f0108 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java @@ -23,10 +23,12 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.List; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.v2.hs.JobHistory; @@ -39,6 +41,10 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.mockito.Mockito.verify; @@ -46,7 +52,9 @@ import org.apache.hadoop.security.authorize.AuthorizationException; import org.apache.hadoop.yarn.logaggregation.AggregatedLogDeletionService; +@RunWith(Parameterized.class) public class TestHSAdminServer { + private boolean securityEnabled = true; private HSAdminServer hsAdminServer = null; private HSAdmin hsAdminClient = null; JobConf conf = null; @@ -80,6 +88,15 @@ public void cacheGroupsAdd(List groups) throws IOException { } } + @Parameters + public static Collection testParameters() { + return Arrays.asList(new Object[][] { { false }, { true } }); + } + + public TestHSAdminServer(boolean enableSecurity) { + securityEnabled = enableSecurity; + } + @Before public void init() throws HadoopIllegalArgumentException, IOException { conf = new JobConf(); @@ -87,6 +104,9 @@ public void init() throws HadoopIllegalArgumentException, IOException { conf.setClass("hadoop.security.group.mapping", MockUnixGroupsMapping.class, GroupMappingServiceProvider.class); conf.setLong("hadoop.security.groups.cache.secs", groupRefreshTimeoutSec); + conf.setBoolean( + CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, + securityEnabled); Groups.getUserToGroupsMappingService(conf); jobHistoryService = mock(JobHistory.class); alds = mock(AggregatedLogDeletionService.class);