big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-9991. Fix Application Tag prefix to userid. Contributed by Szilard Nemeth.

Browse Source
This commit is contained in:
prabhujoseph 2019-11-27 01:43:24 +05:30
parent 8c9018d5c7
commit aa7ab2719f
4 changed files with 42 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -1860,7 +1860,7 @@ public static boolean isAclEnabled(Configuration conf) {
public static final boolean DEFAULT_PROCFS_USE_SMAPS_BASED_RSS_ENABLED =
false;
private static final String APPLICATION_TAG_BASED_PLACEMENT_PREFIX =
"application-tag-based-placement";
RM_PREFIX + "application-tag-based-placement";
public static final String APPLICATION_TAG_BASED_PLACEMENT_ENABLED =
APPLICATION_TAG_BASED_PLACEMENT_PREFIX + ".enable";
public static final boolean DEFAULT_APPLICATION_TAG_BASED_PLACEMENT_ENABLED =

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -1291,11 +1291,12 @@
<property>
<description>
Whether to enable application placement based on user ID passed via
application tags. When it is enabled, u=&lt;userId&gt; pattern will be checked
and if found, the application will be placed onto the found user's queue,
application tags. When it is enabled, userid=&lt;userId&gt;
pattern will be checked and if found, the application will be placed
onto the found user's queue,
if the original user has enough rights on the passed user's queue.
</description>
<name>application-tag-based-placement.enable</name>
<name>yarn.resourcemanager.application-tag-based-placement.enable</name>
<value>false</value>
</property>
@ -1304,7 +1305,7 @@
Comma separated list of users who can use the application tag based
placement, if it is enabled.
</description>
<name>application-tag-based-placement.username.whitelist</name>
<name>yarn.resourcemanager.application-tag-based-placement.username.whitelist</name>
<value></value>
</property>

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
View File

@ -105,6 +105,8 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
private boolean nodeLabelsEnabled;
private Set<String> exclusiveEnforcedPartitions;
private static final String USER_ID_PREFIX = "userid=";
public RMAppManager(RMContext context,
YarnScheduler scheduler, ApplicationMasterService masterService,
ApplicationACLsManager applicationACLsManager, Configuration conf) {
@ -938,11 +940,11 @@ protected String getUserNameForPlacement(final String user,
return usernameUsedForPlacement;
}
LOG.debug("Application tag based placement is enabled, checking for " +
"userId in the application tag");
"'userid' among the application tags");
Set<String> applicationTags = context.getApplicationTags();
String userNameFromAppTag = getUserNameFromApplicationTag(applicationTags);
if (userNameFromAppTag != null) {
LOG.debug("Found userId '{}' in application tag", userNameFromAppTag);
LOG.debug("Found 'userid' '{}' in application tag", userNameFromAppTag);
UserGroupInformation callerUGI = UserGroupInformation
.createRemoteUser(userNameFromAppTag);
// check if the actual user has rights to submit application to the
@ -958,7 +960,7 @@ protected String getUserNameForPlacement(final String user,
userNameFromAppTag, queue, user);
}
} else {
LOG.warn("userId was not found in application tags");
LOG.warn("'userid' was not found in application tags");
}
return usernameUsedForPlacement;
}
@ -979,9 +981,8 @@ private boolean isWhitelistedUser(final String user,
}
private String getUserNameFromApplicationTag(Set<String> applicationTags) {
String userIdPrefix = "u=";
for (String tag: applicationTags) {
if (tag.startsWith(userIdPrefix)) {
if (tag.startsWith(USER_ID_PREFIX)) {
String[] userIdTag = tag.split("=");
if (userIdTag.length == 2) {
return userIdTag[1];

55
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
View File

@ -125,6 +125,8 @@ public class TestAppManager extends AppManagerTestBase{
private static String USER = "user_";
private static String USER0 = USER + 0;
private static final String USER_ID_PREFIX = "userid=";
public synchronized RMAppEventType getAppEventType() {
return appEventType;
}
@ -1205,55 +1207,55 @@ public void testGetUserNameForPlacementTagBasedPlacementDisabled()
throws YarnException {
String user = "user1";
String expectedQueue = "user1Queue";
String userIdTag = "u=user2";
String userIdTag = USER_ID_PREFIX + "user2";
setApplicationTags("tag1", userIdTag, "tag2");
verifyPlacementUsername(expectedQueue, user, user);
}
@Test
/**
* Test case for when the application tag based placement is enabled and
* the submitting user 'user1' is whitelisted and the user from the
* application tag has access to queue.
* Expected behaviour: the placement is done for user from the tag 'user2'
*/
@Test
public void testGetUserNameForPlacementTagBasedPlacementEnabled()
throws YarnException {
String user = "user1";
String expectedQueue = "user1Queue";
String expectedUser = "user2";
String userIdTag = "u=" + expectedUser;
String userIdTag = USER_ID_PREFIX + expectedUser;
setApplicationTags("tag1", userIdTag, "tag2");
enableApplicationTagPlacement(true, user);
verifyPlacementUsername(expectedQueue, user, expectedUser);
}
@Test
/**
* Test case for when the application tag based placement is enabled and
* the submitting user 'user1' is whitelisted and there are multiple valid
* Test case for when the application tag based placement is enabled.
* And submitting user 'user1' is whitelisted and there are multiple valid
* username tags passed
* Expected behaviour: the placement is done for the first valid username
* from the tag 'user2'
*/
@Test
public void testGetUserNameForPlacementTagBasedPlacementMultipleUserIds()
throws YarnException {
String user = "user1";
String expectedQueue = "user1Queue";
String expectedUser = "user2";
String userIdTag = "u=" + expectedUser;
String userIdTag2 = "u=user3";
String userIdTag = USER_ID_PREFIX + expectedUser;
String userIdTag2 = USER_ID_PREFIX + "user3";
setApplicationTags("tag1", userIdTag, "tag2", userIdTag2);
enableApplicationTagPlacement(true, user);
verifyPlacementUsername(expectedQueue, user, expectedUser);
}
@Test
/**
* Test case for when the application tag based placement is enabled but
* no username is set in the application tag
* Test case for when the application tag based placement is enabled.
* And no username is set in the application tag
* Expected behaviour: the placement is done for the submitting user 'user1'
*/
@Test
public void testGetUserNameForPlacementTagBasedPlacementNoUserId()
throws YarnException {
String user = "user1";
@ -1263,87 +1265,90 @@ public void testGetUserNameForPlacementTagBasedPlacementNoUserId()
verifyPlacementUsername(expectedQueue, user, user);
}
@Test
/**
* Test case for when the application tag based placement is enabled but
* the user from the application tag 'user2' does not have access to the
* queue.
* Expected behaviour: the placement is done for the submitting user 'user1'
*/
@Test
public void testGetUserNameForPlacementUserWithoutAccessToQueue()
throws YarnException {
String user = "user1";
String expectedQueue = "user1Queue";
String userIdTag = "u=user2";
String userIdTag = USER_ID_PREFIX + "user2";
setApplicationTags("tag1", userIdTag, "tag2");
enableApplicationTagPlacement(false, user);
verifyPlacementUsername(expectedQueue, user, user);
}
@Test
/**
* Test case for when the application tag based placement is enabled but
* the submitting user 'user1' is not whitelisted and there is a valid
* username tag passed.
* Expected behaviour: the placement is done for the submitting user 'user1'
*/
@Test
public void testGetUserNameForPlacementNotWhitelistedUser()
throws YarnException {
String user = "user1";
String expectedQueue = "user1Queue";
String userIdTag = "u=user2";
String userIdTag = USER_ID_PREFIX + "user2";
setApplicationTags("tag1", userIdTag, "tag2");
enableApplicationTagPlacement(true, "someUser");
verifyPlacementUsername(expectedQueue, user, user);
}
@Test
/**
* Test case for when the application tag based placement is enabled but
* there is no whitelisted user.
* Expected behaviour: the placement is done for the submitting user 'user1'
*/
@Test
public void testGetUserNameForPlacementEmptyWhiteList()
throws YarnException {
String user = "user1";
String expectedQueue = "user1Queue";
String userIdTag = "u=user2";
String userIdTag = USER_ID_PREFIX + "user2";
setApplicationTags("tag1", userIdTag, "tag2");
enableApplicationTagPlacement(false);
verifyPlacementUsername(expectedQueue, user, user);
}
@Test
/**
* Test case for when the application tag based placement is enabled and
* there is one wrongly qualified user 'u=' and a valid user 'u=user2' passed
* via application tag.
* there is one wrongly qualified user
* 'userid=' and a valid user 'userid=user2' passed
* with application tag.
* Expected behaviour: the placement is done for the first valid username
* from the tag 'user2'
*/
@Test
public void testGetUserNameForPlacementWronglyQualifiedFirstUserNameInTag()
throws YarnException {
String user = "user1";
String expectedQueue = "user1Queue";
String expectedUser = "user2";
String userIdTag = "u=" + expectedUser;
String wrongUserIdTag = "u=";
String userIdTag = USER_ID_PREFIX + expectedUser;
String wrongUserIdTag = USER_ID_PREFIX;
setApplicationTags("tag1", wrongUserIdTag, userIdTag, "tag2");
enableApplicationTagPlacement(true, user);
verifyPlacementUsername(expectedQueue, user, expectedUser);
}
@Test
/**
* Test case for when the application tag based placement is enabled and
* there is only one wrongly qualified user 'u=' passed via application tag.
* there is only one wrongly qualified user 'userid=' passed
* with application tag.
* Expected behaviour: the placement is done for the submitting user 'user1'
*/
@Test
public void testGetUserNameForPlacementWronglyQualifiedUserNameInTag()
throws YarnException {
String user = "user1";
String expectedQueue = "user1Queue";
String wrongUserIdTag = "u=";
String wrongUserIdTag = USER_ID_PREFIX;
setApplicationTags("tag1", wrongUserIdTag, "tag2");
enableApplicationTagPlacement(true, user);
verifyPlacementUsername(expectedQueue, user, user);