big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-2715. Fixed ResourceManager to respect common configurations for proxy users/groups beyond just the YARN level config. Contributed by Zhijie Shen.

Browse Source
This commit is contained in:
Vinod Kumar Vavilapalli 2014-10-21 20:09:40 -07:00
parent b2942762d7
commit c0e034336c
9 changed files with 184 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -622,6 +622,9 @@ Release 2.6.0 - UNRELEASED
test-failures: added a dummy in-memory labels-manager. (Wangda Tan via
vinodkv)
YARN-2715. Fixed ResourceManager to respect common configurations for proxy
users/groups beyond just the YARN level config. (Zhijie Shen via vinodkv)
BREAKDOWN OF YARN-1051 SUBTASKS AND RELATED JIRAS
YARN-1707. Introduce APIs to add/remove/resize queues in the

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -31,6 +31,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
@ -560,6 +561,11 @@ public class YarnConfiguration extends Configuration {
public static final String DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION =
"NONE";
/**
* RM proxy users' prefix
*/
public static final String RM_PROXY_USER_PREFIX = RM_PREFIX + "proxyuser.";
////////////////////////////////
// Node Manager Configs
////////////////////////////////

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilterInitializer.java
View File

@ -35,6 +35,7 @@
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
@ -42,14 +43,12 @@
public class RMAuthenticationFilterInitializer extends FilterInitializer {
String configPrefix;
String proxyPrefix;
String signatureSecretFileProperty;
String kerberosPrincipalProperty;
String cookiePath;
public RMAuthenticationFilterInitializer() {
this.configPrefix = "hadoop.http.authentication.";
this.proxyPrefix = "yarn.resourcemanager.webapp.proxyuser.";
this.signatureSecretFileProperty =
AuthenticationFilter.SIGNATURE_SECRET + ".file";
this.kerberosPrincipalProperty = KerberosAuthenticationHandler.PRINCIPAL;
@ -62,15 +61,18 @@ protected Map<String, String> createFilterConfig(Configuration conf) {
// setting the cookie path to root '/' so it is used for all resources.
filterConfig.put(AuthenticationFilter.COOKIE_PATH, cookiePath);
// Before conf object is passed in, RM has already processed it and used RM
// specific configs to overwrite hadoop common ones. Hence we just need to
// source hadoop.proxyuser configs here.
for (Map.Entry<String, String> entry : conf) {
String propName = entry.getKey();
if (propName.startsWith(configPrefix)) {
String value = conf.get(propName);
String name = propName.substring(configPrefix.length());
filterConfig.put(name, value);
} else if (propName.startsWith(proxyPrefix)) {
} else if (propName.startsWith(ProxyUsers.CONF_HADOOP_PROXYUSER)) {
String value = conf.get(propName);
String name = propName.substring("yarn.resourcemanager.webapp.".length());
String name = propName.substring("hadoop.".length());
filterConfig.put(name, value);
}
}

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
View File

@ -421,9 +421,13 @@ public RefreshSuperUserGroupsConfigurationResponse refreshSuperUserGroupsConfigu
throwStandbyException();
}
// Accept hadoop common configs in core-site.xml as well as RM specific
// configurations in yarn-site.xml
Configuration conf =
getConfiguration(new Configuration(false),
YarnConfiguration.CORE_SITE_CONFIGURATION_FILE);
YarnConfiguration.CORE_SITE_CONFIGURATION_FILE,
YarnConfiguration.YARN_SITE_CONFIGURATION_FILE);
RMServerUtils.processRMProxyUsersConf(conf);
ProxyUsers.refreshSuperUserGroupsConfiguration(conf);
RMAuditLogger.logSuccess(user.getShortUserName(),
argName, "AdminService");
@ -594,11 +598,13 @@ public UpdateNodeResourceResponse updateNodeResource(
}
private synchronized Configuration getConfiguration(Configuration conf,
String confFileName) throws YarnException, IOException {
InputStream confFileInputStream = this.rmContext.getConfigurationProvider()
.getConfigurationInputStream(conf, confFileName);
if (confFileInputStream != null) {
conf.addResource(confFileInputStream);
String... confFileNames) throws YarnException, IOException {
for (String confFileName : confFileNames) {
InputStream confFileInputStream = this.rmContext.getConfigurationProvider()
.getConfigurationInputStream(conf, confFileName);
if (confFileInputStream != null) {
conf.addResource(confFileInputStream);
}
}
return conf;
}

27
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
View File

@ -21,12 +21,16 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ContainerId;
@ -36,6 +40,7 @@
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.InvalidContainerReleaseException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceBlacklistRequestException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
@ -248,4 +253,26 @@ public static YarnApplicationAttemptState createApplicationAttemptState(
BuilderUtils.newApplicationResourceUsageReport(-1, -1,
Resources.createResource(-1, -1), Resources.createResource(-1, -1),
Resources.createResource(-1, -1), 0, 0);
/**
* Find all configs whose name starts with
* YarnConfiguration.RM_PROXY_USER_PREFIX, and add a record for each one by
* replacing the prefix with ProxyUsers.CONF_HADOOP_PROXYUSER
*/
public static void processRMProxyUsersConf(Configuration conf) {
Map<String, String> rmProxyUsers = new HashMap<String, String>();
for (Map.Entry<String, String> entry : conf) {
String propName = entry.getKey();
if (propName.startsWith(YarnConfiguration.RM_PROXY_USER_PREFIX)) {
rmProxyUsers.put(ProxyUsers.CONF_HADOOP_PROXYUSER + "." +
propName.substring(YarnConfiguration.RM_PROXY_USER_PREFIX.length()),
entry.getValue());
}
}
for (Map.Entry<String, String> entry : rmProxyUsers.entrySet()) {
conf.set(entry.getKey(), entry.getValue());
}
}
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
View File

@ -23,7 +23,9 @@
import java.net.InetSocketAddress;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
@ -212,6 +214,9 @@ protected void serviceInit(Configuration conf) throws Exception {
.refresh();
// Do refreshSuperUserGroupsConfiguration with loaded core-site.xml
// Or use RM specific configurations to overwrite the common ones first
// if they exist
RMServerUtils.processRMProxyUsersConf(conf);
ProxyUsers.refreshSuperUserGroupsConfiguration(this.conf);
// load yarn-site.xml

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
View File

@ -357,6 +357,24 @@ private void verifyServiceACLsRefresh(ServiceAuthorizationManager manager,
.get("hadoop.proxyuser.test.hosts").size() == 1);
Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyHosts()
.get("hadoop.proxyuser.test.hosts").contains("test_hosts"));
Configuration yarnConf = new Configuration(false);
yarnConf.set("yarn.resourcemanager.proxyuser.test.groups", "test_groups_1");
yarnConf.set("yarn.resourcemanager.proxyuser.test.hosts", "test_hosts_1");
uploadConfiguration(yarnConf, "yarn-site.xml");
// RM specific configs will overwrite the common ones
rm.adminService.refreshSuperUserGroupsConfiguration(
RefreshSuperUserGroupsConfigurationRequest.newInstance());
Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyGroups()
.get("hadoop.proxyuser.test.groups").size() == 1);
Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyGroups()
.get("hadoop.proxyuser.test.groups").contains("test_groups_1"));
Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyHosts()
.get("hadoop.proxyuser.test.hosts").size() == 1);
Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyHosts()
.get("hadoop.proxyuser.test.hosts").contains("test_hosts_1"));
}
@Test

105
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMProxyUsersConf.java Normal file
View File

@ -0,0 +1,105 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager;
import java.util.Arrays;
import java.util.Collection;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AuthorizationException;
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@RunWith(Parameterized.class)
public class TestRMProxyUsersConf {
private static final UserGroupInformation FOO_USER =
UserGroupInformation.createUserForTesting("foo", new String[] { "foo_group" });
private static final UserGroupInformation BAR_USER =
UserGroupInformation.createUserForTesting("bar", new String[] { "bar_group" });
@Parameterized.Parameters
public static Collection<Object[]> headers() {
return Arrays.asList(new Object[][] { { 0 }, { 1 }, { 2 } });
}
private Configuration conf;
public TestRMProxyUsersConf(int round) {
conf = new YarnConfiguration();
switch (round) {
case 0:
// hadoop.proxyuser prefix
conf.set("hadoop.proxyuser.foo.hosts", "localhost");
conf.set("hadoop.proxyuser.foo.users", "bar");
conf.set("hadoop.proxyuser.foo.groups", "bar_group");
break;
case 1:
// yarn.resourcemanager.proxyuser prefix
conf.set("yarn.resourcemanager.proxyuser.foo.hosts", "localhost");
conf.set("yarn.resourcemanager.proxyuser.foo.users", "bar");
conf.set("yarn.resourcemanager.proxyuser.foo.groups", "bar_group");
break;
case 2:
// hadoop.proxyuser prefix has been overwritten by
// yarn.resourcemanager.proxyuser prefix
conf.set("hadoop.proxyuser.foo.hosts", "xyz");
conf.set("hadoop.proxyuser.foo.users", "xyz");
conf.set("hadoop.proxyuser.foo.groups", "xyz");
conf.set("yarn.resourcemanager.proxyuser.foo.hosts", "localhost");
conf.set("yarn.resourcemanager.proxyuser.foo.users", "bar");
conf.set("yarn.resourcemanager.proxyuser.foo.groups", "bar_group");
break;
default:
break;
}
}
@Test
public void testProxyUserConfiguration() throws Exception {
MockRM rm = null;
try {
rm = new MockRM(conf);
rm.start();
// wait for web server starting
Thread.sleep(10000);
UserGroupInformation proxyUser =
UserGroupInformation.createProxyUser(
BAR_USER.getShortUserName(), FOO_USER);
try {
ProxyUsers.getDefaultImpersonationProvider().authorize(proxyUser,
"localhost");
} catch (AuthorizationException e) {
// Exception is not expected
Assert.fail();
}
} finally {
if (rm != null) {
rm.stop();
rm.close();
}
}
}
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokenAuthentication.java
View File

@ -158,8 +158,8 @@ private static void setupAndStartRM() throws Exception {
rmconf.set(YarnConfiguration.NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY,
httpSpnegoKeytabFile.getAbsolutePath());
rmconf.setBoolean("mockrm.webapp.enabled", true);
rmconf.set("yarn.resourcemanager.webapp.proxyuser.client.hosts", "*");
rmconf.set("yarn.resourcemanager.webapp.proxyuser.client.groups", "*");
rmconf.set("yarn.resourcemanager.proxyuser.client.hosts", "*");
rmconf.set("yarn.resourcemanager.proxyuser.client.groups", "*");
UserGroupInformation.setConfiguration(rmconf);
rm = new MockRM(rmconf);
rm.start();