From 6600abbb5c23a83e3a9ef48a945bc8fe19c8178a Mon Sep 17 00:00:00 2001 From: Carlo Curino Date: Fri, 12 May 2017 17:48:55 -0700 Subject: [PATCH] YARN-6587. Refactor of ResourceManager#startWebApp in a Util class. (Giovanni Matteo Fumarola via curino). --- .../resourcemanager/ResourceManager.java | 93 +---------- .../resourcemanager/webapp/RMWebAppUtil.java | 149 ++++++++++++++++++ 2 files changed, 152 insertions(+), 90 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppUtil.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 75d6df292d..1f5e8cd606 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -29,17 +29,13 @@ import org.apache.hadoop.ha.HAServiceProtocol; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.http.HttpServer2; -import org.apache.hadoop.http.lib.StaticUserWebFilter; import org.apache.hadoop.metrics2.MetricsSystem; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.source.JvmMetrics; import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.security.AuthenticationFilterInitializer; import org.apache.hadoop.security.Groups; -import org.apache.hadoop.security.HttpCrossOriginFilterInitializer; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.service.Service; @@ -103,9 +99,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager; import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebApp; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebAppUtil; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; -import org.apache.hadoop.yarn.server.security.http.RMAuthenticationFilter; -import org.apache.hadoop.yarn.server.security.http.RMAuthenticationFilterInitializer; import org.apache.hadoop.yarn.server.webproxy.AppReportFetcher; import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils; import org.apache.hadoop.yarn.server.webproxy.WebAppProxy; @@ -1038,92 +1033,10 @@ public static HttpServer2.Builder httpServerTemplateForRM(Configuration conf, protected void startWepApp() { - // Use the customized yarn filter instead of the standard kerberos filter to - // allow users to authenticate using delegation tokens - // 4 conditions need to be satisfied - - // 1. security is enabled - // 2. http auth type is set to kerberos - // 3. "yarn.resourcemanager.webapp.use-yarn-filter" override is set to true - // 4. hadoop.http.filter.initializers container AuthenticationFilterInitializer - Configuration conf = getConfig(); - boolean enableCorsFilter = - conf.getBoolean(YarnConfiguration.RM_WEBAPP_ENABLE_CORS_FILTER, - YarnConfiguration.DEFAULT_RM_WEBAPP_ENABLE_CORS_FILTER); - boolean useYarnAuthenticationFilter = - conf.getBoolean( - YarnConfiguration.RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER, - YarnConfiguration.DEFAULT_RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER); - String authPrefix = "hadoop.http.authentication."; - String authTypeKey = authPrefix + "type"; - String filterInitializerConfKey = "hadoop.http.filter.initializers"; - String actualInitializers = ""; - Class[] initializersClasses = - conf.getClasses(filterInitializerConfKey); - // setup CORS - if (enableCorsFilter) { - conf.setBoolean(HttpCrossOriginFilterInitializer.PREFIX - + HttpCrossOriginFilterInitializer.ENABLED_SUFFIX, true); - } - - boolean hasHadoopAuthFilterInitializer = false; - boolean hasRMAuthFilterInitializer = false; - if (initializersClasses != null) { - for (Class initializer : initializersClasses) { - if (initializer.getName().equals( - AuthenticationFilterInitializer.class.getName())) { - hasHadoopAuthFilterInitializer = true; - } - if (initializer.getName().equals( - RMAuthenticationFilterInitializer.class.getName())) { - hasRMAuthFilterInitializer = true; - } - } - if (UserGroupInformation.isSecurityEnabled() - && useYarnAuthenticationFilter - && hasHadoopAuthFilterInitializer - && conf.get(authTypeKey, "").equals( - KerberosAuthenticationHandler.TYPE)) { - ArrayList target = new ArrayList(); - for (Class filterInitializer : initializersClasses) { - if (filterInitializer.getName().equals( - AuthenticationFilterInitializer.class.getName())) { - if (hasRMAuthFilterInitializer == false) { - target.add(RMAuthenticationFilterInitializer.class.getName()); - } - continue; - } - target.add(filterInitializer.getName()); - } - actualInitializers = StringUtils.join(",", target); - - LOG.info("Using RM authentication filter(kerberos/delegation-token)" - + " for RM webapp authentication"); - RMAuthenticationFilter - .setDelegationTokenSecretManager(getClientRMService().rmDTSecretManager); - conf.set(filterInitializerConfKey, actualInitializers); - } - } - - // if security is not enabled and the default filter initializer has not - // been set, set the initializer to include the - // RMAuthenticationFilterInitializer which in turn will set up the simple - // auth filter. - - String initializers = conf.get(filterInitializerConfKey); - if (!UserGroupInformation.isSecurityEnabled()) { - if (initializersClasses == null || initializersClasses.length == 0) { - conf.set(filterInitializerConfKey, - RMAuthenticationFilterInitializer.class.getName()); - conf.set(authTypeKey, "simple"); - } else if (initializers.equals(StaticUserWebFilter.class.getName())) { - conf.set(filterInitializerConfKey, - RMAuthenticationFilterInitializer.class.getName() + "," - + initializers); - conf.set(authTypeKey, "simple"); - } - } + RMWebAppUtil.setupSecurityAndFilters(conf, + getClientRMService().rmDTSecretManager); Builder builder = WebApps diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppUtil.java new file mode 100644 index 0000000000..263828b776 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppUtil.java @@ -0,0 +1,149 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import java.util.ArrayList; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.lib.StaticUserWebFilter; +import org.apache.hadoop.security.AuthenticationFilterInitializer; +import org.apache.hadoop.security.HttpCrossOriginFilterInitializer; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager; +import org.apache.hadoop.yarn.server.security.http.RMAuthenticationFilter; +import org.apache.hadoop.yarn.server.security.http.RMAuthenticationFilterInitializer; + +/** + * Util class for ResourceManager WebApp. + */ +public final class RMWebAppUtil { + + private static final Log LOG = LogFactory.getLog(RMWebAppUtil.class); + + /** + * Private constructor. + */ + private RMWebAppUtil() { + // not called + } + + /** + * Helper method to setup filters and authentication for ResourceManager + * WebServices. + * + * Use the customized yarn filter instead of the standard kerberos filter to + * allow users to authenticate using delegation tokens 4 conditions need to be + * satisfied: + * + * 1. security is enabled. + * + * 2. http auth type is set to kerberos. + * + * 3. "yarn.resourcemanager.webapp.use-yarn-filter" override is set to true. + * + * 4. hadoop.http.filter.initializers container + * AuthenticationFilterInitializer. + * + * @param conf RM configuration. + * @param rmDTSecretManager RM specific delegation token secret manager. + **/ + public static void setupSecurityAndFilters(Configuration conf, + RMDelegationTokenSecretManager rmDTSecretManager) { + + boolean enableCorsFilter = + conf.getBoolean(YarnConfiguration.RM_WEBAPP_ENABLE_CORS_FILTER, + YarnConfiguration.DEFAULT_RM_WEBAPP_ENABLE_CORS_FILTER); + boolean useYarnAuthenticationFilter = conf.getBoolean( + YarnConfiguration.RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER, + YarnConfiguration.DEFAULT_RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER); + String authPrefix = "hadoop.http.authentication."; + String authTypeKey = authPrefix + "type"; + String filterInitializerConfKey = "hadoop.http.filter.initializers"; + String actualInitializers = ""; + Class[] initializersClasses = conf.getClasses(filterInitializerConfKey); + + // setup CORS + if (enableCorsFilter) { + conf.setBoolean(HttpCrossOriginFilterInitializer.PREFIX + + HttpCrossOriginFilterInitializer.ENABLED_SUFFIX, true); + } + + boolean hasHadoopAuthFilterInitializer = false; + boolean hasRMAuthFilterInitializer = false; + if (initializersClasses != null) { + for (Class initializer : initializersClasses) { + if (initializer.getName() + .equals(AuthenticationFilterInitializer.class.getName())) { + hasHadoopAuthFilterInitializer = true; + } + if (initializer.getName() + .equals(RMAuthenticationFilterInitializer.class.getName())) { + hasRMAuthFilterInitializer = true; + } + } + if (UserGroupInformation.isSecurityEnabled() + && useYarnAuthenticationFilter && hasHadoopAuthFilterInitializer + && conf.get(authTypeKey, "") + .equals(KerberosAuthenticationHandler.TYPE)) { + ArrayList target = new ArrayList(); + for (Class filterInitializer : initializersClasses) { + if (filterInitializer.getName() + .equals(AuthenticationFilterInitializer.class.getName())) { + if (!hasRMAuthFilterInitializer) { + target.add(RMAuthenticationFilterInitializer.class.getName()); + } + continue; + } + target.add(filterInitializer.getName()); + } + actualInitializers = StringUtils.join(",", target); + + LOG.info("Using RM authentication filter(kerberos/delegation-token)" + + " for RM webapp authentication"); + RMAuthenticationFilter + .setDelegationTokenSecretManager(rmDTSecretManager); + conf.set(filterInitializerConfKey, actualInitializers); + } + } + + // if security is not enabled and the default filter initializer has not + // been set, set the initializer to include the + // RMAuthenticationFilterInitializer which in turn will set up the simple + // auth filter. + + String initializers = conf.get(filterInitializerConfKey); + if (!UserGroupInformation.isSecurityEnabled()) { + if (initializersClasses == null || initializersClasses.length == 0) { + conf.set(filterInitializerConfKey, + RMAuthenticationFilterInitializer.class.getName()); + conf.set(authTypeKey, "simple"); + } else if (initializers.equals(StaticUserWebFilter.class.getName())) { + conf.set(filterInitializerConfKey, + RMAuthenticationFilterInitializer.class.getName() + "," + + initializers); + conf.set(authTypeKey, "simple"); + } + } + } +}