From aa7ab2719f745f6e2a5cfbca713bb49865cf52bd Mon Sep 17 00:00:00 2001 From: prabhujoseph Date: Wed, 27 Nov 2019 01:43:24 +0530 Subject: [PATCH] YARN-9991. Fix Application Tag prefix to userid. Contributed by Szilard Nemeth. --- .../hadoop/yarn/conf/YarnConfiguration.java | 2 +- .../src/main/resources/yarn-default.xml | 9 +-- .../server/resourcemanager/RMAppManager.java | 11 ++-- .../resourcemanager/TestAppManager.java | 55 ++++++++++--------- 4 files changed, 42 insertions(+), 35 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index dea4c0a83e..d23b6301ef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1860,7 +1860,7 @@ public static boolean isAclEnabled(Configuration conf) { public static final boolean DEFAULT_PROCFS_USE_SMAPS_BASED_RSS_ENABLED = false; private static final String APPLICATION_TAG_BASED_PLACEMENT_PREFIX = - "application-tag-based-placement"; + RM_PREFIX + "application-tag-based-placement"; public static final String APPLICATION_TAG_BASED_PLACEMENT_ENABLED = APPLICATION_TAG_BASED_PLACEMENT_PREFIX + ".enable"; public static final boolean DEFAULT_APPLICATION_TAG_BASED_PLACEMENT_ENABLED = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 0e8063e9fe..e53801b58a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1291,11 +1291,12 @@ Whether to enable application placement based on user ID passed via - application tags. When it is enabled, u=<userId> pattern will be checked - and if found, the application will be placed onto the found user's queue, + application tags. When it is enabled, userid=<userId> + pattern will be checked and if found, the application will be placed + onto the found user's queue, if the original user has enough rights on the passed user's queue. - application-tag-based-placement.enable + yarn.resourcemanager.application-tag-based-placement.enable false @@ -1304,7 +1305,7 @@ Comma separated list of users who can use the application tag based placement, if it is enabled. - application-tag-based-placement.username.whitelist + yarn.resourcemanager.application-tag-based-placement.username.whitelist diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 30c5a8dbec..df29a0a9a4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -105,6 +105,8 @@ public class RMAppManager implements EventHandler, private boolean nodeLabelsEnabled; private Set exclusiveEnforcedPartitions; + private static final String USER_ID_PREFIX = "userid="; + public RMAppManager(RMContext context, YarnScheduler scheduler, ApplicationMasterService masterService, ApplicationACLsManager applicationACLsManager, Configuration conf) { @@ -938,11 +940,11 @@ protected String getUserNameForPlacement(final String user, return usernameUsedForPlacement; } LOG.debug("Application tag based placement is enabled, checking for " + - "userId in the application tag"); + "'userid' among the application tags"); Set applicationTags = context.getApplicationTags(); String userNameFromAppTag = getUserNameFromApplicationTag(applicationTags); if (userNameFromAppTag != null) { - LOG.debug("Found userId '{}' in application tag", userNameFromAppTag); + LOG.debug("Found 'userid' '{}' in application tag", userNameFromAppTag); UserGroupInformation callerUGI = UserGroupInformation .createRemoteUser(userNameFromAppTag); // check if the actual user has rights to submit application to the @@ -958,7 +960,7 @@ protected String getUserNameForPlacement(final String user, userNameFromAppTag, queue, user); } } else { - LOG.warn("userId was not found in application tags"); + LOG.warn("'userid' was not found in application tags"); } return usernameUsedForPlacement; } @@ -979,9 +981,8 @@ private boolean isWhitelistedUser(final String user, } private String getUserNameFromApplicationTag(Set applicationTags) { - String userIdPrefix = "u="; for (String tag: applicationTags) { - if (tag.startsWith(userIdPrefix)) { + if (tag.startsWith(USER_ID_PREFIX)) { String[] userIdTag = tag.split("="); if (userIdTag.length == 2) { return userIdTag[1]; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index 07dea743be..7d68b2d4ed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -125,6 +125,8 @@ public class TestAppManager extends AppManagerTestBase{ private static String USER = "user_"; private static String USER0 = USER + 0; + private static final String USER_ID_PREFIX = "userid="; + public synchronized RMAppEventType getAppEventType() { return appEventType; } @@ -1205,55 +1207,55 @@ public void testGetUserNameForPlacementTagBasedPlacementDisabled() throws YarnException { String user = "user1"; String expectedQueue = "user1Queue"; - String userIdTag = "u=user2"; + String userIdTag = USER_ID_PREFIX + "user2"; setApplicationTags("tag1", userIdTag, "tag2"); verifyPlacementUsername(expectedQueue, user, user); } - @Test /** * Test case for when the application tag based placement is enabled and * the submitting user 'user1' is whitelisted and the user from the * application tag has access to queue. * Expected behaviour: the placement is done for user from the tag 'user2' */ + @Test public void testGetUserNameForPlacementTagBasedPlacementEnabled() throws YarnException { String user = "user1"; String expectedQueue = "user1Queue"; String expectedUser = "user2"; - String userIdTag = "u=" + expectedUser; + String userIdTag = USER_ID_PREFIX + expectedUser; setApplicationTags("tag1", userIdTag, "tag2"); enableApplicationTagPlacement(true, user); verifyPlacementUsername(expectedQueue, user, expectedUser); } - @Test /** - * Test case for when the application tag based placement is enabled and - * the submitting user 'user1' is whitelisted and there are multiple valid + * Test case for when the application tag based placement is enabled. + * And submitting user 'user1' is whitelisted and there are multiple valid * username tags passed * Expected behaviour: the placement is done for the first valid username * from the tag 'user2' */ + @Test public void testGetUserNameForPlacementTagBasedPlacementMultipleUserIds() throws YarnException { String user = "user1"; String expectedQueue = "user1Queue"; String expectedUser = "user2"; - String userIdTag = "u=" + expectedUser; - String userIdTag2 = "u=user3"; + String userIdTag = USER_ID_PREFIX + expectedUser; + String userIdTag2 = USER_ID_PREFIX + "user3"; setApplicationTags("tag1", userIdTag, "tag2", userIdTag2); enableApplicationTagPlacement(true, user); verifyPlacementUsername(expectedQueue, user, expectedUser); } - @Test /** - * Test case for when the application tag based placement is enabled but - * no username is set in the application tag + * Test case for when the application tag based placement is enabled. + * And no username is set in the application tag * Expected behaviour: the placement is done for the submitting user 'user1' */ + @Test public void testGetUserNameForPlacementTagBasedPlacementNoUserId() throws YarnException { String user = "user1"; @@ -1263,87 +1265,90 @@ public void testGetUserNameForPlacementTagBasedPlacementNoUserId() verifyPlacementUsername(expectedQueue, user, user); } - @Test /** * Test case for when the application tag based placement is enabled but * the user from the application tag 'user2' does not have access to the * queue. * Expected behaviour: the placement is done for the submitting user 'user1' */ + @Test public void testGetUserNameForPlacementUserWithoutAccessToQueue() throws YarnException { String user = "user1"; String expectedQueue = "user1Queue"; - String userIdTag = "u=user2"; + String userIdTag = USER_ID_PREFIX + "user2"; setApplicationTags("tag1", userIdTag, "tag2"); enableApplicationTagPlacement(false, user); verifyPlacementUsername(expectedQueue, user, user); } - @Test /** * Test case for when the application tag based placement is enabled but * the submitting user 'user1' is not whitelisted and there is a valid * username tag passed. * Expected behaviour: the placement is done for the submitting user 'user1' */ + @Test public void testGetUserNameForPlacementNotWhitelistedUser() throws YarnException { String user = "user1"; String expectedQueue = "user1Queue"; - String userIdTag = "u=user2"; + String userIdTag = USER_ID_PREFIX + "user2"; setApplicationTags("tag1", userIdTag, "tag2"); enableApplicationTagPlacement(true, "someUser"); verifyPlacementUsername(expectedQueue, user, user); } - @Test /** * Test case for when the application tag based placement is enabled but * there is no whitelisted user. * Expected behaviour: the placement is done for the submitting user 'user1' */ + @Test public void testGetUserNameForPlacementEmptyWhiteList() throws YarnException { String user = "user1"; String expectedQueue = "user1Queue"; - String userIdTag = "u=user2"; + String userIdTag = USER_ID_PREFIX + "user2"; setApplicationTags("tag1", userIdTag, "tag2"); enableApplicationTagPlacement(false); verifyPlacementUsername(expectedQueue, user, user); } - @Test + /** * Test case for when the application tag based placement is enabled and - * there is one wrongly qualified user 'u=' and a valid user 'u=user2' passed - * via application tag. + * there is one wrongly qualified user + * 'userid=' and a valid user 'userid=user2' passed + * with application tag. * Expected behaviour: the placement is done for the first valid username * from the tag 'user2' */ + @Test public void testGetUserNameForPlacementWronglyQualifiedFirstUserNameInTag() throws YarnException { String user = "user1"; String expectedQueue = "user1Queue"; String expectedUser = "user2"; - String userIdTag = "u=" + expectedUser; - String wrongUserIdTag = "u="; + String userIdTag = USER_ID_PREFIX + expectedUser; + String wrongUserIdTag = USER_ID_PREFIX; setApplicationTags("tag1", wrongUserIdTag, userIdTag, "tag2"); enableApplicationTagPlacement(true, user); verifyPlacementUsername(expectedQueue, user, expectedUser); } - @Test /** * Test case for when the application tag based placement is enabled and - * there is only one wrongly qualified user 'u=' passed via application tag. + * there is only one wrongly qualified user 'userid=' passed + * with application tag. * Expected behaviour: the placement is done for the submitting user 'user1' */ + @Test public void testGetUserNameForPlacementWronglyQualifiedUserNameInTag() throws YarnException { String user = "user1"; String expectedQueue = "user1Queue"; - String wrongUserIdTag = "u="; + String wrongUserIdTag = USER_ID_PREFIX; setApplicationTags("tag1", wrongUserIdTag, "tag2"); enableApplicationTagPlacement(true, user); verifyPlacementUsername(expectedQueue, user, user);