big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10330. Add missing test scenarios to TestUserGroupMappingPlacementRule and TestAppNameMappingPlacementRule. Contributed by Peter Bacsko

Browse Source
This commit is contained in:
Szilard Nemeth 2020-07-01 17:42:45 +02:00
parent cfb2084cba
commit 439c51425e
3 changed files with 278 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestAppNameMappingPlacementRule.java
View File

@ -37,7 +37,15 @@
import static org.mockito.Mockito.when;
public class TestAppNameMappingPlacementRule {
private static final String ROOT_QUEUE = "root";
private static final String Q2_QUEUE = "q2";
private static final String Q1_QUEUE = "q1";
private static final String USER_NAME = "user";
private static final String DEFAULT_QUEUE = "default";
private static final String APPLICATION_PLACEHOLDER = "%application";
private static final String AMBIGUOUS_QUEUE = "ambiguousQueue";
private static final String APP_NAME = "DistributedShell";
private static final String MAPREDUCE_APP_NAME = "MAPREDUCE";
private YarnConfiguration conf = new YarnConfiguration();
@ -62,16 +70,20 @@ private void verifyQueueMapping(QueueMapping queueMapping,
CapacitySchedulerQueueManager qm =
mock(CapacitySchedulerQueueManager.class);
when(qm.isAmbiguous(Mockito.isA(String.class))).thenReturn(false);
when(qm.isAmbiguous(AMBIGUOUS_QUEUE)).thenReturn(true);
rule.queueManager = qm;
ApplicationSubmissionContext asc = Records.newRecord(
ApplicationSubmissionContext.class);
if (inputQueue.equals("%application")) {
if (inputQueue.equals(APPLICATION_PLACEHOLDER)) {
inputQueue = APP_NAME;
}
asc.setQueue(inputQueue);
String appName = queueMapping.getSource();
if (appName.equals("%application")) {
// to create a scenario when source != appName
if (appName.equals(APPLICATION_PLACEHOLDER)
|| appName.equals(MAPREDUCE_APP_NAME)) {
appName = APP_NAME;
}
asc.setApplicationName(appName);
@ -81,31 +93,85 @@ private void verifyQueueMapping(QueueMapping queueMapping,
ctx != null ? ctx.getQueue() : inputQueue);
}
public QueueMapping queueMappingBuilder(String source, String queue) {
public QueueMapping getQueueMapping(String source, String queue) {
return getQueueMapping(source, null, queue);
}
public QueueMapping getQueueMapping(String source, String parent,
String queue) {
return QueueMapping.QueueMappingBuilder.create()
.type(QueueMapping.MappingType.APPLICATION)
.source(source)
.queue(queue)
.parentQueue(parent)
.build();
}
@Test
public void testMapping() throws YarnException {
// simple base case for mapping user to queue
verifyQueueMapping(queueMappingBuilder(APP_NAME,
"q1"), "user_1", "q1");
verifyQueueMapping(queueMappingBuilder("%application", "q2"), "user_1",
"q2");
verifyQueueMapping(queueMappingBuilder("%application", "%application"),
"user_1", APP_NAME);
// specify overwritten, and see if user specified a queue, and it will be
// overridden
verifyQueueMapping(queueMappingBuilder(APP_NAME,
"q1"), "1", "q2", "q1", true);
// if overwritten not specified, it should be which user specified
verifyQueueMapping(queueMappingBuilder(APP_NAME,
"q1"), "1", "q2", "q2", false);
public void testSpecificAppNameMappedToDefinedQueue() throws YarnException {
verifyQueueMapping(getQueueMapping(APP_NAME, Q1_QUEUE),
USER_NAME, Q1_QUEUE);
}
@Test
public void testPlaceholderAppSourceMappedToQueue() throws YarnException {
verifyQueueMapping(getQueueMapping(APPLICATION_PLACEHOLDER, Q2_QUEUE),
USER_NAME, Q2_QUEUE);
}
@Test
public void testPlaceHolderAppSourceAndQueueMappedToAppNameQueue()
throws YarnException {
verifyQueueMapping(getQueueMapping(APPLICATION_PLACEHOLDER,
APPLICATION_PLACEHOLDER), USER_NAME, APP_NAME);
}
@Test
public void testQueueInMappingOverridesSpecifiedQueue()
throws YarnException {
verifyQueueMapping(getQueueMapping(APP_NAME,
Q1_QUEUE), USER_NAME, Q2_QUEUE, Q1_QUEUE, true);
}
@Test
public void testQueueInMappingDoesNotOverrideSpecifiedQueue()
throws YarnException {
verifyQueueMapping(getQueueMapping(APP_NAME,
Q1_QUEUE), USER_NAME, Q2_QUEUE, Q2_QUEUE, false);
}
@Test
public void testDefaultQueueInMappingIsNotUsedWithoutOverride()
throws YarnException {
verifyQueueMapping(getQueueMapping(APP_NAME,
DEFAULT_QUEUE), USER_NAME, Q2_QUEUE, Q2_QUEUE, false);
}
@Test
public void testDefaultQueueInMappingEqualsToInputQueue()
throws YarnException {
verifyQueueMapping(getQueueMapping(APP_NAME,
DEFAULT_QUEUE), USER_NAME, DEFAULT_QUEUE, DEFAULT_QUEUE, false);
}
@Test
public void testMappingSourceDiffersFromInputQueue() throws YarnException {
verifyQueueMapping(getQueueMapping(MAPREDUCE_APP_NAME,
Q1_QUEUE), USER_NAME, DEFAULT_QUEUE, DEFAULT_QUEUE, false);
}
@Test(expected = YarnException.class)
public void testMappingContainsAmbiguousLeafQueueWithoutParent()
throws YarnException {
verifyQueueMapping(getQueueMapping(APP_NAME, AMBIGUOUS_QUEUE),
USER_NAME, DEFAULT_QUEUE, DEFAULT_QUEUE, false);
}
@Test
public void testMappingContainsAmbiguousLeafQueueWithParent()
throws YarnException {
verifyQueueMapping(getQueueMapping(APP_NAME, ROOT_QUEUE, AMBIGUOUS_QUEUE),
USER_NAME, DEFAULT_QUEUE, AMBIGUOUS_QUEUE, false);
}
}

194
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java
View File

@ -149,6 +149,7 @@ private AbstractCSQueue createRootQueue(String rootQueueName) {
ParentQueue root = mock(ParentQueue.class);
when(root.getQueuePath()).thenReturn(rootQueueName);
when(queueManager.getQueue(rootQueueName)).thenReturn(root);
when(queueManager.getQueueByFullName(rootQueueName)).thenReturn(root);
return root;
}
@ -220,11 +221,13 @@ private void verifyQueueMapping(QueueMappingTestData queueMappingTestData)
.withQueue("root.agroup.a")
.withQueue("root.asubgroup2")
.withQueue("root.bsubgroup2.b")
.withQueue("root.users.primarygrouponly")
.withQueue("root.admins.primarygrouponly")
.withManagedParentQueue("root.managedParent")
.build();
when(queueManager.getQueue(isNull())).thenReturn(null);
when(queueManager.isAmbiguous("primarygrouponly")).thenReturn(true);
rule.setQueueManager(queueManager);
ApplicationSubmissionContext asc = Records.newRecord(
ApplicationSubmissionContext.class);
@ -344,16 +347,185 @@ public void testMapping() throws YarnException {
.build());
verifyQueueMapping(
QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER)
.source("%user")
.queue("%user")
.parentQueue("%primary_group")
.build())
.inputUser("a")
.expectedQueue("a")
.expectedParentQueue("root.agroup")
.build());
.queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER)
.source("%user")
.queue("%user")
.parentQueue("%primary_group")
.build())
.inputUser("a")
.expectedQueue("a")
.expectedParentQueue("root.agroup")
.build());
}
@Test
public void testUserMappingToPrimaryGroupInvalidNestedPlaceholder()
throws YarnException {
// u:%user:%primary_group.%random, no matching queue
verifyQueueMapping(
QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER)
.source("%user")
.queue("%random")
.parentQueue("%primary_group")
.build())
.inputUser("a")
.expectedQueue("default")
.build());
}
@Test
public void testUserMappingToSecondaryGroupInvalidNestedPlaceholder()
throws YarnException {
// u:%user:%secondary_group.%random, no matching queue
verifyQueueMapping(
QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER)
.source("%user")
.queue("%random")
.parentQueue("%secondary_group")
.build())
.inputUser("a")
.expectedQueue("default")
.build());
}
@Test
public void testUserMappingDiffersFromSubmitterQueueDoesNotExist()
throws YarnException {
// u:a:%random, submitter: xyz, no matching queue
verifyQueueMapping(
QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER)
.source("a")
.queue("%random")
.build())
.inputUser("xyz")
.expectedQueue("default")
.build());
}
@Test
public void testSpecificUserMappingToPrimaryGroup() throws YarnException {
// u:a:%primary_group
verifyQueueMapping(
QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER)
.source("a")
.queue("%primary_group")
.build())
.inputUser("a")
.expectedQueue("agroup")
.build());
}
@Test
public void testSpecificUserMappingToSecondaryGroup()
throws YarnException {
// u:a:%secondary_group
verifyQueueMapping(
QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER)
.source("a")
.queue("%secondary_group")
.build())
.inputUser("a")
.expectedQueue("asubgroup2")
.build());
}
@Test
public void testSpecificUserMappingWithNoSecondaryGroup()
throws YarnException {
// u:nosecondarygroupuser:%secondary_group, no matching queue
verifyQueueMapping(
QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER)
.source("nosecondarygroupuser")
.queue("%secondary_group")
.build())
.inputUser("nosecondarygroupuser")
.expectedQueue("default")
.build());
}
@Test
public void testGenericUserMappingWithNoSecondaryGroup()
throws YarnException {
// u:%user:%user, no matching queue
verifyQueueMapping(
QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER)
.source("%user")
.queue("%user")
.parentQueue("%secondary_group")
.build())
.inputUser("nosecondarygroupuser")
.expectedQueue("default")
.build());
}
@Test(expected = YarnException.class)
public void testUserMappingToNestedUserPrimaryGroupWithAmbiguousQueues()
throws YarnException {
// u:%user:%user, submitter nosecondarygroupuser, queue is ambiguous
verifyQueueMapping(
QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER)
.source("%user")
.queue("%user")
.parentQueue("%primary_group")
.build())
.inputUser("nosecondarygroupuser")
.build());
}
@Test(expected = YarnException.class)
public void testResolvedQueueIsNotManaged()
throws YarnException {
// u:%user:%primary_group.%user, "admins" group will be "root",
// resulting parent queue will be "root" which is not managed
verifyQueueMapping(
QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER)
.source("%user")
.queue("%user")
.parentQueue("%primary_group")
.build())
.inputUser("admins")
.build());
}
@Test(expected = YarnException.class)
public void testUserMappingToPrimaryGroupWithAmbiguousQueues()
throws YarnException {
// u:%user:%primary_group, submitter nosecondarygroupuser,
// queue is ambiguous
verifyQueueMapping(
QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER)
.source("%user")
.queue("%primary_group")
.build())
.inputUser("nosecondarygroupuser")
.expectedQueue("default")
.build());
}
@Test
public void testUserMappingToQueueNamedAsUsernameWithSecondaryGroupAsParentQueue()
throws YarnException {
verifyQueueMapping(
QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java
View File

@ -28,7 +28,14 @@ public class SimpleGroupsMapping implements GroupMappingServiceProvider {
@Override
public List<String> getGroups(String user) {
return Arrays.asList(user + "group", user + "subgroup1", user + "subgroup2");
if ("admins".equals(user)) {
return Arrays.asList("root");
} else if ("nosecondarygroupuser".equals(user)) {
return Arrays.asList("primarygrouponly");
} else {
return Arrays.asList(
user + "group", user + "subgroup1", user + "subgroup2");
}
}
@Override