From 439c51425e7095d8b2cde058d3576bda4b3c2182 Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Wed, 1 Jul 2020 17:42:45 +0200 Subject: [PATCH] YARN-10330. Add missing test scenarios to TestUserGroupMappingPlacementRule and TestAppNameMappingPlacementRule. Contributed by Peter Bacsko --- .../TestAppNameMappingPlacementRule.java | 106 ++++++++-- .../TestUserGroupMappingPlacementRule.java | 194 +++++++++++++++++- .../scheduler/fair/SimpleGroupsMapping.java | 11 +- 3 files changed, 278 insertions(+), 33 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestAppNameMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestAppNameMappingPlacementRule.java index 1204213cf9..29141aedf6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestAppNameMappingPlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestAppNameMappingPlacementRule.java @@ -37,7 +37,15 @@ import static org.mockito.Mockito.when; public class TestAppNameMappingPlacementRule { + private static final String ROOT_QUEUE = "root"; + private static final String Q2_QUEUE = "q2"; + private static final String Q1_QUEUE = "q1"; + private static final String USER_NAME = "user"; + private static final String DEFAULT_QUEUE = "default"; + private static final String APPLICATION_PLACEHOLDER = "%application"; + private static final String AMBIGUOUS_QUEUE = "ambiguousQueue"; private static final String APP_NAME = "DistributedShell"; + private static final String MAPREDUCE_APP_NAME = "MAPREDUCE"; private YarnConfiguration conf = new YarnConfiguration(); @@ -62,16 +70,20 @@ private void verifyQueueMapping(QueueMapping queueMapping, CapacitySchedulerQueueManager qm = mock(CapacitySchedulerQueueManager.class); when(qm.isAmbiguous(Mockito.isA(String.class))).thenReturn(false); + when(qm.isAmbiguous(AMBIGUOUS_QUEUE)).thenReturn(true); + rule.queueManager = qm; ApplicationSubmissionContext asc = Records.newRecord( ApplicationSubmissionContext.class); - if (inputQueue.equals("%application")) { + if (inputQueue.equals(APPLICATION_PLACEHOLDER)) { inputQueue = APP_NAME; } asc.setQueue(inputQueue); String appName = queueMapping.getSource(); - if (appName.equals("%application")) { + // to create a scenario when source != appName + if (appName.equals(APPLICATION_PLACEHOLDER) + || appName.equals(MAPREDUCE_APP_NAME)) { appName = APP_NAME; } asc.setApplicationName(appName); @@ -81,31 +93,85 @@ private void verifyQueueMapping(QueueMapping queueMapping, ctx != null ? ctx.getQueue() : inputQueue); } - public QueueMapping queueMappingBuilder(String source, String queue) { + public QueueMapping getQueueMapping(String source, String queue) { + return getQueueMapping(source, null, queue); + } + + public QueueMapping getQueueMapping(String source, String parent, + String queue) { return QueueMapping.QueueMappingBuilder.create() .type(QueueMapping.MappingType.APPLICATION) .source(source) .queue(queue) + .parentQueue(parent) .build(); } @Test - public void testMapping() throws YarnException { - // simple base case for mapping user to queue - verifyQueueMapping(queueMappingBuilder(APP_NAME, - "q1"), "user_1", "q1"); - verifyQueueMapping(queueMappingBuilder("%application", "q2"), "user_1", - "q2"); - verifyQueueMapping(queueMappingBuilder("%application", "%application"), - "user_1", APP_NAME); - - // specify overwritten, and see if user specified a queue, and it will be - // overridden - verifyQueueMapping(queueMappingBuilder(APP_NAME, - "q1"), "1", "q2", "q1", true); - - // if overwritten not specified, it should be which user specified - verifyQueueMapping(queueMappingBuilder(APP_NAME, - "q1"), "1", "q2", "q2", false); + public void testSpecificAppNameMappedToDefinedQueue() throws YarnException { + verifyQueueMapping(getQueueMapping(APP_NAME, Q1_QUEUE), + USER_NAME, Q1_QUEUE); } + + @Test + public void testPlaceholderAppSourceMappedToQueue() throws YarnException { + verifyQueueMapping(getQueueMapping(APPLICATION_PLACEHOLDER, Q2_QUEUE), + USER_NAME, Q2_QUEUE); + } + + @Test + public void testPlaceHolderAppSourceAndQueueMappedToAppNameQueue() + throws YarnException { + verifyQueueMapping(getQueueMapping(APPLICATION_PLACEHOLDER, + APPLICATION_PLACEHOLDER), USER_NAME, APP_NAME); + } + + @Test + public void testQueueInMappingOverridesSpecifiedQueue() + throws YarnException { + verifyQueueMapping(getQueueMapping(APP_NAME, + Q1_QUEUE), USER_NAME, Q2_QUEUE, Q1_QUEUE, true); + } + + @Test + public void testQueueInMappingDoesNotOverrideSpecifiedQueue() + throws YarnException { + verifyQueueMapping(getQueueMapping(APP_NAME, + Q1_QUEUE), USER_NAME, Q2_QUEUE, Q2_QUEUE, false); + } + + @Test + public void testDefaultQueueInMappingIsNotUsedWithoutOverride() + throws YarnException { + verifyQueueMapping(getQueueMapping(APP_NAME, + DEFAULT_QUEUE), USER_NAME, Q2_QUEUE, Q2_QUEUE, false); + } + + @Test + public void testDefaultQueueInMappingEqualsToInputQueue() + throws YarnException { + verifyQueueMapping(getQueueMapping(APP_NAME, + DEFAULT_QUEUE), USER_NAME, DEFAULT_QUEUE, DEFAULT_QUEUE, false); + } + + @Test + public void testMappingSourceDiffersFromInputQueue() throws YarnException { + verifyQueueMapping(getQueueMapping(MAPREDUCE_APP_NAME, + Q1_QUEUE), USER_NAME, DEFAULT_QUEUE, DEFAULT_QUEUE, false); + } + + @Test(expected = YarnException.class) + public void testMappingContainsAmbiguousLeafQueueWithoutParent() + throws YarnException { + verifyQueueMapping(getQueueMapping(APP_NAME, AMBIGUOUS_QUEUE), + USER_NAME, DEFAULT_QUEUE, DEFAULT_QUEUE, false); + } + + @Test + public void testMappingContainsAmbiguousLeafQueueWithParent() + throws YarnException { + verifyQueueMapping(getQueueMapping(APP_NAME, ROOT_QUEUE, AMBIGUOUS_QUEUE), + USER_NAME, DEFAULT_QUEUE, AMBIGUOUS_QUEUE, false); + } + } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java index e436b6e1b8..06a3e4a869 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java @@ -149,6 +149,7 @@ private AbstractCSQueue createRootQueue(String rootQueueName) { ParentQueue root = mock(ParentQueue.class); when(root.getQueuePath()).thenReturn(rootQueueName); when(queueManager.getQueue(rootQueueName)).thenReturn(root); + when(queueManager.getQueueByFullName(rootQueueName)).thenReturn(root); return root; } @@ -220,11 +221,13 @@ private void verifyQueueMapping(QueueMappingTestData queueMappingTestData) .withQueue("root.agroup.a") .withQueue("root.asubgroup2") .withQueue("root.bsubgroup2.b") + .withQueue("root.users.primarygrouponly") + .withQueue("root.admins.primarygrouponly") .withManagedParentQueue("root.managedParent") .build(); when(queueManager.getQueue(isNull())).thenReturn(null); - + when(queueManager.isAmbiguous("primarygrouponly")).thenReturn(true); rule.setQueueManager(queueManager); ApplicationSubmissionContext asc = Records.newRecord( ApplicationSubmissionContext.class); @@ -344,16 +347,185 @@ public void testMapping() throws YarnException { .build()); verifyQueueMapping( QueueMappingTestDataBuilder.create() - .queueMapping(QueueMappingBuilder.create() - .type(MappingType.USER) - .source("%user") - .queue("%user") - .parentQueue("%primary_group") - .build()) - .inputUser("a") - .expectedQueue("a") - .expectedParentQueue("root.agroup") - .build()); + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.USER) + .source("%user") + .queue("%user") + .parentQueue("%primary_group") + .build()) + .inputUser("a") + .expectedQueue("a") + .expectedParentQueue("root.agroup") + .build()); + } + + @Test + public void testUserMappingToPrimaryGroupInvalidNestedPlaceholder() + throws YarnException { + // u:%user:%primary_group.%random, no matching queue + verifyQueueMapping( + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.USER) + .source("%user") + .queue("%random") + .parentQueue("%primary_group") + .build()) + .inputUser("a") + .expectedQueue("default") + .build()); + } + + @Test + public void testUserMappingToSecondaryGroupInvalidNestedPlaceholder() + throws YarnException { + // u:%user:%secondary_group.%random, no matching queue + verifyQueueMapping( + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.USER) + .source("%user") + .queue("%random") + .parentQueue("%secondary_group") + .build()) + .inputUser("a") + .expectedQueue("default") + .build()); + } + + @Test + public void testUserMappingDiffersFromSubmitterQueueDoesNotExist() + throws YarnException { + // u:a:%random, submitter: xyz, no matching queue + verifyQueueMapping( + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.USER) + .source("a") + .queue("%random") + .build()) + .inputUser("xyz") + .expectedQueue("default") + .build()); + } + + @Test + public void testSpecificUserMappingToPrimaryGroup() throws YarnException { + // u:a:%primary_group + verifyQueueMapping( + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.USER) + .source("a") + .queue("%primary_group") + .build()) + .inputUser("a") + .expectedQueue("agroup") + .build()); + } + + @Test + public void testSpecificUserMappingToSecondaryGroup() + throws YarnException { + // u:a:%secondary_group + verifyQueueMapping( + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.USER) + .source("a") + .queue("%secondary_group") + .build()) + .inputUser("a") + .expectedQueue("asubgroup2") + .build()); + } + + @Test + public void testSpecificUserMappingWithNoSecondaryGroup() + throws YarnException { + // u:nosecondarygroupuser:%secondary_group, no matching queue + verifyQueueMapping( + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.USER) + .source("nosecondarygroupuser") + .queue("%secondary_group") + .build()) + .inputUser("nosecondarygroupuser") + .expectedQueue("default") + .build()); + } + + @Test + public void testGenericUserMappingWithNoSecondaryGroup() + throws YarnException { + // u:%user:%user, no matching queue + verifyQueueMapping( + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.USER) + .source("%user") + .queue("%user") + .parentQueue("%secondary_group") + .build()) + .inputUser("nosecondarygroupuser") + .expectedQueue("default") + .build()); + } + + @Test(expected = YarnException.class) + public void testUserMappingToNestedUserPrimaryGroupWithAmbiguousQueues() + throws YarnException { + // u:%user:%user, submitter nosecondarygroupuser, queue is ambiguous + verifyQueueMapping( + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.USER) + .source("%user") + .queue("%user") + .parentQueue("%primary_group") + .build()) + .inputUser("nosecondarygroupuser") + .build()); + } + + @Test(expected = YarnException.class) + public void testResolvedQueueIsNotManaged() + throws YarnException { + // u:%user:%primary_group.%user, "admins" group will be "root", + // resulting parent queue will be "root" which is not managed + verifyQueueMapping( + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.USER) + .source("%user") + .queue("%user") + .parentQueue("%primary_group") + .build()) + .inputUser("admins") + .build()); + } + + @Test(expected = YarnException.class) + public void testUserMappingToPrimaryGroupWithAmbiguousQueues() + throws YarnException { + // u:%user:%primary_group, submitter nosecondarygroupuser, + // queue is ambiguous + verifyQueueMapping( + QueueMappingTestDataBuilder.create() + .queueMapping(QueueMappingBuilder.create() + .type(MappingType.USER) + .source("%user") + .queue("%primary_group") + .build()) + .inputUser("nosecondarygroupuser") + .expectedQueue("default") + .build()); + } + + @Test + public void testUserMappingToQueueNamedAsUsernameWithSecondaryGroupAsParentQueue() + throws YarnException { verifyQueueMapping( QueueMappingTestDataBuilder.create() .queueMapping(QueueMappingBuilder.create() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java index f7648c86d4..9c916e3641 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java @@ -25,10 +25,17 @@ import org.apache.hadoop.security.GroupMappingServiceProvider; public class SimpleGroupsMapping implements GroupMappingServiceProvider { - + @Override public List getGroups(String user) { - return Arrays.asList(user + "group", user + "subgroup1", user + "subgroup2"); + if ("admins".equals(user)) { + return Arrays.asList("root"); + } else if ("nosecondarygroupuser".equals(user)) { + return Arrays.asList("primarygrouponly"); + } else { + return Arrays.asList( + user + "group", user + "subgroup1", user + "subgroup2"); + } } @Override