From 24fa232707b56f6ccc88a460dc0d708d35a28ff4 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Mon, 3 Feb 2014 22:10:56 +0000 Subject: [PATCH] YARN-1667. Modified RM HA handling of super users (with proxying ability) to be available across RM failover by making using of a remote configuration-provider. Contributed by Xuan Gong. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564100 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop/security/authorize/ProxyUsers.java | 11 ++++ hadoop-yarn-project/CHANGES.txt | 33 ++++++++++ .../hadoop/yarn/conf/YarnConfiguration.java | 3 + .../server/resourcemanager/AdminService.java | 15 ++--- .../resourcemanager/TestRMAdminService.java | 61 +++++++++++++++++++ 5 files changed, 116 insertions(+), 7 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java index 341285e1a7..5295258873 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ProxyUsers.java @@ -30,6 +30,8 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; +import com.google.common.annotations.VisibleForTesting; + @InterfaceAudience.Private public class ProxyUsers { @@ -177,4 +179,13 @@ private static boolean isWildcardList(Collection list) { (list.contains("*")); } + @VisibleForTesting + public static Map> getProxyGroups() { + return proxyGroups; + } + + @VisibleForTesting + public static Map> getProxyHosts() { + return proxyHosts; + } } diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 9a947f713c..e4b8900843 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -114,6 +114,39 @@ Release 2.4.0 - UNRELEASED failover by making using of a remote configuration-provider. (Xuan Gong via vinodkv) + YARN-1667. Modified RM HA handling of super users (with proxying ability) to + be available across RM failover by making using of a remote + configuration-provider. (Xuan Gong via vinodkv) + + OPTIMIZATIONS + + BUG FIXES + + YARN-935. Correcting pom.xml to build applicationhistoryserver module + successfully. (Zhijie Shen via vinodkv) + + YARN-962. Fixed bug in application-history proto file and renamed it be just + a client proto file. (Zhijie Shen via vinodkv) + + YARN-984. Renamed the incorrectly named applicationhistoryservice.records.pb.impl + package to be applicationhistoryservice.records.impl.pb. (Devaraj K via vinodkv) + + YARN-1534. Fixed failure of test TestAHSWebApp. (Shinichi Yamashita via vinodkv) + + YARN-1555. Fixed test failures in applicationhistoryservice.* (Vinod Kumar + Vavilapalli via mayank) + + YARN-1594. Updated pom.xml of applicationhistoryservice sub-project according to + YARN-888. (Vinod Kumar Vavilapalli via zjshen) + + YARN-1596. Fixed Javadoc warnings on branch YARN-321. (Vinod Kumar Vavilapalli + via zjshen) + + YARN-1597. Fixed Findbugs warnings on branch YARN-321. (Vinod Kumar Vavilapalli + via zjshen) + + YARN-1595. Made enabling history service configurable and fixed test failures on + branch YARN-321. (Vinod Kumar Vavilapalli via zjshen) OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index af385f81db..919ed90143 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -43,6 +43,9 @@ public class YarnConfiguration extends Configuration { @Private public static final String YARN_SITE_XML_FILE = "yarn-site.xml"; + @Private + public static final String CORE_SITE_CONFIGURATION_FILE = "core-site.xml"; + private static final String YARN_DEFAULT_XML_FILE = "yarn-default.xml"; static { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index 9a33b706f9..3bfd47d137 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -363,21 +363,22 @@ public RefreshNodesResponse refreshNodes(RefreshNodesRequest request) @Override public RefreshSuperUserGroupsConfigurationResponse refreshSuperUserGroupsConfiguration( RefreshSuperUserGroupsConfigurationRequest request) - throws YarnException, StandbyException { - UserGroupInformation user = checkAcls("refreshSuperUserGroupsConfiguration"); + throws YarnException, IOException { + String argName = "refreshSuperUserGroupsConfiguration"; + UserGroupInformation user = checkAcls(argName); - // TODO (YARN-1459): Revisit handling super-user-groups on Standby RM if (!isRMActive()) { - RMAuditLogger.logFailure(user.getShortUserName(), - "refreshSuperUserGroupsConfiguration", + RMAuditLogger.logFailure(user.getShortUserName(), argName, adminAcl.toString(), "AdminService", "ResourceManager is not active. Can not refresh super-user-groups."); throwStandbyException(); } - ProxyUsers.refreshSuperUserGroupsConfiguration(new Configuration()); + Configuration conf = + getConfiguration(YarnConfiguration.CORE_SITE_CONFIGURATION_FILE); + ProxyUsers.refreshSuperUserGroupsConfiguration(conf); RMAuditLogger.logSuccess(user.getShortUserName(), - "refreshSuperUserGroupsConfiguration", "AdminService"); + argName, "AdminService"); return recordFactory.newRecordInstance( RefreshSuperUserGroupsConfigurationResponse.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java index 4b7018528f..797b422684 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -29,10 +29,12 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.junit.After; @@ -188,6 +190,65 @@ public void testAdminAclsWithFileSystemBasedConfigurationProvider() Assert.assertEquals(aclStringAfter, "world:anyone:rwcda"); } + @Test + public void + testRefreshSuperUserGroupsWithLocalConfigurationProvider() { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + + try { + rm.adminService.refreshSuperUserGroupsConfiguration( + RefreshSuperUserGroupsConfigurationRequest.newInstance()); + } catch (Exception ex) { + fail("Using localConfigurationProvider. Should not get any exception."); + } + } + + @Test + public void + testRefreshSuperUserGroupsWithFileSystemBasedConfigurationProvider() + throws IOException, YarnException { + configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + + // clean the remoteDirectory + cleanRemoteDirectory(); + + try { + rm.adminService.refreshSuperUserGroupsConfiguration( + RefreshSuperUserGroupsConfigurationRequest.newInstance()); + fail("FileSystemBasedConfigurationProvider is used." + + " Should get an exception here"); + } catch (Exception ex) { + Assert.assertTrue(ex.getMessage().contains( + "Can not find Configuration: core-site.xml")); + } + + Configuration coreConf = new Configuration(false); + coreConf.set("hadoop.proxyuser.test.groups", "test_groups"); + coreConf.set("hadoop.proxyuser.test.hosts", "test_hosts"); + String coreConfFile = writeConfigurationXML(coreConf, + "core-site.xml"); + + // upload the file into Remote File System + uploadToRemoteFileSystem(new Path(coreConfFile)); + rm.adminService.refreshSuperUserGroupsConfiguration( + RefreshSuperUserGroupsConfigurationRequest.newInstance()); + Assert.assertTrue(ProxyUsers.getProxyGroups() + .get("hadoop.proxyuser.test.groups").size() == 1); + Assert.assertTrue(ProxyUsers.getProxyGroups() + .get("hadoop.proxyuser.test.groups").contains("test_groups")); + + Assert.assertTrue(ProxyUsers.getProxyHosts() + .get("hadoop.proxyuser.test.hosts").size() == 1); + Assert.assertTrue(ProxyUsers.getProxyHosts() + .get("hadoop.proxyuser.test.hosts").contains("test_hosts")); + } + private String writeConfigurationXML(Configuration conf, String confXMLName) throws IOException { DataOutputStream output = null;