big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10321. Break down TestUserGroupMappingPlacementRule#testMapping into test scenarios

Browse Source 
Contributed by Szilard Nemeth. Reviewed by Gergely Pollak.
This commit is contained in:
Prabhu Joseph 2020-06-22 12:47:46 +05:30 committed by Prabhu Joseph
parent 17ffcab5f6
commit ce1008fe61
2 changed files with 284 additions and 214 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
View File

@ -141,7 +141,7 @@ private ApplicationPlacementContext getPlacementForUser(String user)
} else { } else {
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("Creating placement context for user {} using " + LOG.debug("Creating placement context for user {} using " +
"static user static mapping", user); "current user static mapping", user);
} }
return getPlacementContext(mapping); return getPlacementContext(mapping);
} }
@ -173,7 +173,7 @@ private ApplicationPlacementContext getPlacementForUser(String user)
} else { } else {
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("Creating placement context for user {} using " + LOG.debug("Creating placement context for user {} using " +
"static user static mapping", user); "current user static mapping", user);
} }
return getPlacementContext(mapping); return getPlacementContext(mapping);
} }

494
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java
View File

@ -169,7 +169,7 @@ public void testSecondaryGroupMapping() throws YarnException {
} }
@Test @Test
public void testNullGroupMapping() throws YarnException { public void testNullGroupMapping() {
conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
NullGroupsMapping.class, GroupMappingServiceProvider.class); NullGroupsMapping.class, GroupMappingServiceProvider.class);
try { try {
@ -190,246 +190,316 @@ public void testNullGroupMapping() throws YarnException {
} }
@Test @Test
public void testMapping() throws YarnException { public void testSimpleUserMappingToSpecificQueue() throws YarnException {
//if a mapping rule defines no parent, we cannot expect auto creation,
// so we must provide already existing queues
verifyQueueMapping( verifyQueueMapping(
QueueMappingTestDataBuilder.create() QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create() .queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER) .type(MappingType.USER)
.source("a") .source("a")
.queue("a") .queue("a")
.build()) .build())
.inputUser("a") .inputUser("a")
.expectedQueue("a") .expectedQueue("a")
.build()); .build());
verifyQueueMapping( }
QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()
.type(MappingType.GROUP)
.source("agroup")
.queue("a")
.build())
.inputUser("a")
.expectedQueue("a")
.build());
verifyQueueMapping(
QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER)
.source("%user")
.queue("b")
.build())
.inputUser("a")
.expectedQueue("b")
.build());
verifyQueueMapping(
QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER)
.source("%user")
.queue("%user")
.build())
.inputUser("a")
.expectedQueue("a")
.build());
verifyQueueMapping(
QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER)
.source("%user")
.queue("%primary_group")
.build())
.inputUser("a")
.expectedQueue("agroup")
.expectedParentQueue("root")
.build());
verifyQueueMapping(
QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER)
.source("%user")
.queue("%user")
.parentQueue("%primary_group")
.build())
.inputUser("a")
.expectedQueue("a")
.expectedParentQueue("root.agroup")
.build());
verifyQueueMapping(
QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER)
.source("%user")
.queue("%user")
.parentQueue("%secondary_group")
.build())
.inputUser("b")
.expectedQueue("b")
.expectedParentQueue("root.bsubgroup2")
.build());
verifyQueueMapping(
QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()
.type(MappingType.GROUP)
.source("asubgroup1")
.queue("a")
.build())
.inputUser("a")
.expectedQueue("a")
.build());
// "agroup" queue exists @Test
public void testSimpleGroupMappingToSpecificQueue() throws YarnException {
verifyQueueMapping( verifyQueueMapping(
QueueMappingTestDataBuilder.create() QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create() .queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER) .type(MappingType.GROUP)
.source("%user") .source("agroup")
.queue("%primary_group") .queue("a")
.parentQueue("root") .build())
.build()) .inputUser("a")
.inputUser("a") .expectedQueue("a")
.expectedQueue("agroup") .build());
.expectedParentQueue("root") }
.build());
@Test
public void testUserMappingToSpecificQueueForEachUser() throws YarnException {
verifyQueueMapping(
QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER)
.source("%user")
.queue("b")
.build())
.inputUser("a")
.expectedQueue("b")
.build());
}
@Test
public void testUserMappingToQueueNamedAsUsername() throws YarnException {
verifyQueueMapping(
QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER)
.source("%user")
.queue("%user")
.build())
.inputUser("a")
.expectedQueue("a")
.build());
}
@Test
public void testUserMappingToQueueNamedGroupOfTheUser() throws YarnException {
verifyQueueMapping(
QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER)
.source("%user")
.queue("%primary_group")
.build())
.inputUser("a")
.expectedQueue("agroup")
.expectedParentQueue("root")
.build());
}
@Test
public void testUserMappingToQueueNamedAsUsernameWithPrimaryGroupAsParentQueue()
throws YarnException {
verifyQueueMapping(
QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER)
.source("%user")
.queue("%user")
.parentQueue("%primary_group")
.build())
.inputUser("a")
.expectedQueue("a")
.expectedParentQueue("root.agroup")
.build());
}
@Test
public void testUserMappingToQueueNamedAsUsernameWithSecondaryGroupAsParentQueue()
throws YarnException {
verifyQueueMapping(
QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER)
.source("%user")
.queue("%user")
.parentQueue("%secondary_group")
.build())
.inputUser("b")
.expectedQueue("b")
.expectedParentQueue("root.bsubgroup2")
.build());
}
@Test
public void testGroupMappingToStaticQueue() throws YarnException {
verifyQueueMapping(
QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()
.type(MappingType.GROUP)
.source("asubgroup1")
.queue("a")
.build())
.inputUser("a")
.expectedQueue("a")
.build());
}
@Test
public void testUserMappingToQueueNamedAsGroupNameWithRootAsParentQueue()
throws YarnException {
verifyQueueMapping(
QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER)
.source("%user")
.queue("%primary_group")
.parentQueue("root")
.build())
.inputUser("a")
.expectedQueue("agroup")
.expectedParentQueue("root")
.build());
}
@Test
public void testUserMappingToPrimaryGroupQueueDoesNotExistUnmanagedParent()
throws YarnException {
// "abcgroup" queue doesn't exist, %primary_group queue, not managed parent // "abcgroup" queue doesn't exist, %primary_group queue, not managed parent
verifyQueueMapping( verifyQueueMapping(
QueueMappingTestDataBuilder.create() QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create() .queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER) .type(MappingType.USER)
.source("%user") .source("%user")
.queue("%primary_group") .queue("%primary_group")
.parentQueue("bsubgroup2") .parentQueue("bsubgroup2")
.build()) .build())
.inputUser("abc") .inputUser("abc")
.expectedQueue("default") .expectedQueue("default")
.build()); .build());
}
@Test
public void testUserMappingToPrimaryGroupQueueDoesNotExistManagedParent()
throws YarnException {
// "abcgroup" queue doesn't exist, %primary_group queue, managed parent // "abcgroup" queue doesn't exist, %primary_group queue, managed parent
verifyQueueMapping( verifyQueueMapping(
QueueMappingTestDataBuilder.create() QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create() .queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER) .type(MappingType.USER)
.source("%user") .source("%user")
.queue("%primary_group") .queue("%primary_group")
.parentQueue("managedParent") .parentQueue("managedParent")
.build()) .build())
.inputUser("abc") .inputUser("abc")
.expectedQueue("abcgroup") .expectedQueue("abcgroup")
.expectedParentQueue("root.managedParent") .expectedParentQueue("root.managedParent")
.build()); .build());
}
@Test
public void testUserMappingToSecondaryGroupQueueDoesNotExist()
throws YarnException {
// "abcgroup" queue doesn't exist, %secondary_group queue // "abcgroup" queue doesn't exist, %secondary_group queue
verifyQueueMapping( verifyQueueMapping(
QueueMappingTestDataBuilder.create() QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create() .queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER) .type(MappingType.USER)
.source("%user") .source("%user")
.queue("%secondary_group") .queue("%secondary_group")
.parentQueue("bsubgroup2") .parentQueue("bsubgroup2")
.build()) .build())
.inputUser("abc") .inputUser("abc")
.expectedQueue("default") .expectedQueue("default")
.build()); .build());
}
@Test
public void testUserMappingToSecondaryGroupQueueUnderParent()
throws YarnException {
// "asubgroup2" queue exists, %secondary_group queue // "asubgroup2" queue exists, %secondary_group queue
verifyQueueMapping( verifyQueueMapping(
QueueMappingTestDataBuilder.create() QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create() .queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER) .type(MappingType.USER)
.source("%user") .source("%user")
.queue("%secondary_group") .queue("%secondary_group")
.parentQueue("root") .parentQueue("root")
.build()) .build())
.inputUser("a") .inputUser("a")
.expectedQueue("asubgroup2") .expectedQueue("asubgroup2")
.expectedParentQueue("root") .expectedParentQueue("root")
.build()); .build());
}
@Test
public void testUserMappingToSpecifiedQueueOverwritesInputQueueFromMapping()
throws YarnException {
// specify overwritten, and see if user specified a queue, and it will be // specify overwritten, and see if user specified a queue, and it will be
// overridden // overridden
verifyQueueMapping( verifyQueueMapping(
QueueMappingTestDataBuilder.create() QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create() .queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER) .type(MappingType.USER)
.source("user") .source("user")
.queue("a") .queue("a")
.build()) .build())
.inputUser("user") .inputUser("user")
.inputQueue("b") .inputQueue("b")
.expectedQueue("a") .expectedQueue("a")
.overwrite(true) .overwrite(true)
.build()); .build());
}
// if overwritten not specified, it should be which user specified @Test
verifyQueueMapping( public void testUserMappingToExplicitlySpecifiedQueue() throws YarnException {
QueueMappingTestDataBuilder.create() // if overwritten not specified, it should be which user specified
.queueMapping(QueueMappingBuilder.create() verifyQueueMapping(
.type(MappingType.USER) QueueMappingTestDataBuilder.create()
.source("user") .queueMapping(QueueMappingBuilder.create()
.queue("a") .type(MappingType.USER)
.build()) .source("user")
.inputUser("user") .queue("a")
.inputQueue("b") .build())
.expectedQueue("b") .inputUser("user")
.build()); .inputQueue("b")
.expectedQueue("b")
.build());
}
// if overwritten not specified, it should be which user specified @Test
verifyQueueMapping( public void testGroupMappingToExplicitlySpecifiedQueue()
QueueMappingTestDataBuilder.create() throws YarnException {
.queueMapping(QueueMappingBuilder.create() // if overwritten not specified, it should be which user specified
.type(MappingType.GROUP) verifyQueueMapping(
.source("usergroup") QueueMappingTestDataBuilder.create()
.queue("%user") .queueMapping(QueueMappingBuilder.create()
.parentQueue("usergroup") .type(MappingType.GROUP)
.build()) .source("usergroup")
.inputUser("user") .queue("%user")
.inputQueue("a") .parentQueue("usergroup")
.expectedQueue("a") .build())
.build()); .inputUser("user")
.inputQueue("a")
.expectedQueue("a")
.build());
}
// if overwritten not specified, it should be which user specified @Test
verifyQueueMapping( public void testGroupMappingToSpecifiedQueueOverwritesInputQueueFromMapping()
QueueMappingTestDataBuilder.create() throws YarnException {
.queueMapping(QueueMappingBuilder.create() // if overwritten not specified, it should be which user specified
.type(MappingType.GROUP) verifyQueueMapping(
.source("usergroup") QueueMappingTestDataBuilder.create()
.queue("b") .queueMapping(QueueMappingBuilder.create()
.parentQueue("root.bsubgroup2") .type(MappingType.GROUP)
.build()) .source("usergroup")
.inputUser("user") .queue("b")
.inputQueue("a") .parentQueue("root.bsubgroup2")
.expectedQueue("b") .build())
.overwrite(true) .inputUser("user")
.build()); .inputQueue("a")
.expectedQueue("b")
.overwrite(true)
.build());
}
// If user specific queue is enabled for a specified group under a given @Test
// parent queue public void testGroupMappingToSpecifiedQueueUnderAGivenParentQueue()
verifyQueueMapping( throws YarnException {
QueueMappingTestDataBuilder.create() // If user specific queue is enabled for a specified group under a given
.queueMapping(QueueMappingBuilder.create() // parent queue
.type(MappingType.GROUP) verifyQueueMapping(
.source("agroup") QueueMappingTestDataBuilder.create()
.queue("%user") .queueMapping(QueueMappingBuilder.create()
.parentQueue("root.agroup") .type(MappingType.GROUP)
.build()) .source("agroup")
.inputUser("a") .queue("%user")
.expectedQueue("a") .parentQueue("root.agroup")
.build()); .build())
.inputUser("a")
.expectedQueue("a")
.build());
}
// If user specific queue is enabled for a specified group without parent @Test
// queue public void testGroupMappingToSpecifiedQueueWithoutParentQueue()
verifyQueueMapping( throws YarnException {
QueueMappingTestDataBuilder.create() // If user specific queue is enabled for a specified group without parent
.queueMapping(QueueMappingBuilder.create() // queue
.type(MappingType.GROUP) verifyQueueMapping(
.source("agroup") QueueMappingTestDataBuilder.create()
.queue("%user") .queueMapping(QueueMappingBuilder.create()
.build()) .type(MappingType.GROUP)
.inputUser("a") .source("agroup")
.expectedQueue("a") .queue("%user")
.build()); .build())
.inputUser("a")
.expectedQueue("a")
.build());
} }
/** /**