big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-1667. Modified RM HA handling of super users (with proxying ability) to be available across RM failover by making using of a remote configuration-provider. Contributed by Xuan Gong.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564100 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2014-02-03 22:10:56 +00:00
parent eff1e809f2
commit 24fa232707
5 changed files with 116 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java
View File

@ -30,6 +30,8 @@
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
import com.google.common.annotations.VisibleForTesting;
@InterfaceAudience.Private
public class ProxyUsers {
@ -177,4 +179,13 @@ private static boolean isWildcardList(Collection<String> list) {
(list.contains("*"));
}
@VisibleForTesting
public static Map<String, Collection<String>> getProxyGroups() {
return proxyGroups;
}
@VisibleForTesting
public static Map<String, Collection<String>> getProxyHosts() {
return proxyHosts;
}
}

33
hadoop-yarn-project/CHANGES.txt
View File

@ -114,6 +114,39 @@ Release 2.4.0 - UNRELEASED
failover by making using of a remote configuration-provider. (Xuan Gong via
vinodkv)
YARN-1667. Modified RM HA handling of super users (with proxying ability) to
be available across RM failover by making using of a remote
configuration-provider. (Xuan Gong via vinodkv)
OPTIMIZATIONS
BUG FIXES
YARN-935. Correcting pom.xml to build applicationhistoryserver module
successfully. (Zhijie Shen via vinodkv)
YARN-962. Fixed bug in application-history proto file and renamed it be just
a client proto file. (Zhijie Shen via vinodkv)
YARN-984. Renamed the incorrectly named applicationhistoryservice.records.pb.impl
package to be applicationhistoryservice.records.impl.pb. (Devaraj K via vinodkv)
YARN-1534. Fixed failure of test TestAHSWebApp. (Shinichi Yamashita via vinodkv)
YARN-1555. Fixed test failures in applicationhistoryservice.* (Vinod Kumar
Vavilapalli via mayank)
YARN-1594. Updated pom.xml of applicationhistoryservice sub-project according to
YARN-888. (Vinod Kumar Vavilapalli via zjshen)
YARN-1596. Fixed Javadoc warnings on branch YARN-321. (Vinod Kumar Vavilapalli
via zjshen)
YARN-1597. Fixed Findbugs warnings on branch YARN-321. (Vinod Kumar Vavilapalli
via zjshen)
YARN-1595. Made enabling history service configurable and fixed test failures on
branch YARN-321. (Vinod Kumar Vavilapalli via zjshen)
OPTIMIZATIONS
BUG FIXES

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -43,6 +43,9 @@ public class YarnConfiguration extends Configuration {
@Private
public static final String YARN_SITE_XML_FILE = "yarn-site.xml";
@Private
public static final String CORE_SITE_CONFIGURATION_FILE = "core-site.xml";
private static final String YARN_DEFAULT_XML_FILE = "yarn-default.xml";
static {

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
View File

@ -363,21 +363,22 @@ public RefreshNodesResponse refreshNodes(RefreshNodesRequest request)
@Override
public RefreshSuperUserGroupsConfigurationResponse refreshSuperUserGroupsConfiguration(
RefreshSuperUserGroupsConfigurationRequest request)
throws YarnException, StandbyException {
UserGroupInformation user = checkAcls("refreshSuperUserGroupsConfiguration");
throws YarnException, IOException {
String argName = "refreshSuperUserGroupsConfiguration";
UserGroupInformation user = checkAcls(argName);
// TODO (YARN-1459): Revisit handling super-user-groups on Standby RM
if (!isRMActive()) {
RMAuditLogger.logFailure(user.getShortUserName(),
"refreshSuperUserGroupsConfiguration",
RMAuditLogger.logFailure(user.getShortUserName(), argName,
adminAcl.toString(), "AdminService",
"ResourceManager is not active. Can not refresh super-user-groups.");
throwStandbyException();
}
ProxyUsers.refreshSuperUserGroupsConfiguration(new Configuration());
Configuration conf =
getConfiguration(YarnConfiguration.CORE_SITE_CONFIGURATION_FILE);
ProxyUsers.refreshSuperUserGroupsConfiguration(conf);
RMAuditLogger.logSuccess(user.getShortUserName(),
"refreshSuperUserGroupsConfiguration", "AdminService");
argName, "AdminService");
return recordFactory.newRecordInstance(
RefreshSuperUserGroupsConfigurationResponse.class);

61
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
View File

@ -29,10 +29,12 @@
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.junit.After;
@ -188,6 +190,65 @@ public void testAdminAclsWithFileSystemBasedConfigurationProvider()
Assert.assertEquals(aclStringAfter, "world:anyone:rwcda");
}
@Test
public void
testRefreshSuperUserGroupsWithLocalConfigurationProvider() {
rm = new MockRM(configuration);
rm.init(configuration);
rm.start();
try {
rm.adminService.refreshSuperUserGroupsConfiguration(
RefreshSuperUserGroupsConfigurationRequest.newInstance());
} catch (Exception ex) {
fail("Using localConfigurationProvider. Should not get any exception.");
}
}
@Test
public void
testRefreshSuperUserGroupsWithFileSystemBasedConfigurationProvider()
throws IOException, YarnException {
configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
"org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
rm = new MockRM(configuration);
rm.init(configuration);
rm.start();
// clean the remoteDirectory
cleanRemoteDirectory();
try {
rm.adminService.refreshSuperUserGroupsConfiguration(
RefreshSuperUserGroupsConfigurationRequest.newInstance());
fail("FileSystemBasedConfigurationProvider is used." +
" Should get an exception here");
} catch (Exception ex) {
Assert.assertTrue(ex.getMessage().contains(
"Can not find Configuration: core-site.xml"));
}
Configuration coreConf = new Configuration(false);
coreConf.set("hadoop.proxyuser.test.groups", "test_groups");
coreConf.set("hadoop.proxyuser.test.hosts", "test_hosts");
String coreConfFile = writeConfigurationXML(coreConf,
"core-site.xml");
// upload the file into Remote File System
uploadToRemoteFileSystem(new Path(coreConfFile));
rm.adminService.refreshSuperUserGroupsConfiguration(
RefreshSuperUserGroupsConfigurationRequest.newInstance());
Assert.assertTrue(ProxyUsers.getProxyGroups()
.get("hadoop.proxyuser.test.groups").size() == 1);
Assert.assertTrue(ProxyUsers.getProxyGroups()
.get("hadoop.proxyuser.test.groups").contains("test_groups"));
Assert.assertTrue(ProxyUsers.getProxyHosts()
.get("hadoop.proxyuser.test.hosts").size() == 1);
Assert.assertTrue(ProxyUsers.getProxyHosts()
.get("hadoop.proxyuser.test.hosts").contains("test_hosts"));
}
private String writeConfigurationXML(Configuration conf, String confXMLName)
throws IOException {
DataOutputStream output = null;