big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10254. CapacityScheduler incorrect User Group Mapping after leaf queue change. Contributed by Gergely Pollak

Browse Source
This commit is contained in:
Szilard Nemeth 2020-06-02 18:32:06 +02:00
parent 910d88eeed
commit f65f64e8ae
3 changed files with 213 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

217
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
View File

@ -105,35 +105,76 @@ private ApplicationPlacementContext getPlacementForUser(String user)
if (mapping.getParentQueue() != null if (mapping.getParentQueue() != null
&& mapping.getParentQueue().equals(PRIMARY_GROUP_MAPPING) && mapping.getParentQueue().equals(PRIMARY_GROUP_MAPPING)
&& mapping.getQueue().equals(CURRENT_USER_MAPPING)) { && mapping.getQueue().equals(CURRENT_USER_MAPPING)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Creating placement context for user {} using " +
"primary group current user mapping", user);
}
return getContextForGroupParent(user, mapping, return getContextForGroupParent(user, mapping,
getPrimaryGroup(user)); getPrimaryGroup(user));
} else if (mapping.getParentQueue() != null } else if (mapping.getParentQueue() != null
&& mapping.getParentQueue().equals(SECONDARY_GROUP_MAPPING) && mapping.getParentQueue().equals(SECONDARY_GROUP_MAPPING)
&& mapping.getQueue().equals(CURRENT_USER_MAPPING)) { && mapping.getQueue().equals(CURRENT_USER_MAPPING)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Creating placement context for user {} using " +
"secondary group current user mapping", user);
}
return getContextForGroupParent(user, mapping, return getContextForGroupParent(user, mapping,
getSecondaryGroup(user)); getSecondaryGroup(user));
} else if (mapping.getQueue().equals(CURRENT_USER_MAPPING)) { } else if (mapping.getQueue().equals(CURRENT_USER_MAPPING)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Creating placement context for user {} using " +
"current user mapping", user);
}
return getPlacementContext(mapping, user); return getPlacementContext(mapping, user);
} else if (mapping.getQueue().equals(PRIMARY_GROUP_MAPPING)) { } else if (mapping.getQueue().equals(PRIMARY_GROUP_MAPPING)) {
return getContextForPrimaryGroup(user, mapping); if (LOG.isDebugEnabled()) {
LOG.debug("Creating placement context for user {} using " +
"primary group mapping", user);
}
return getPlacementContext(mapping, getPrimaryGroup(user));
} else if (mapping.getQueue().equals(SECONDARY_GROUP_MAPPING)) { } else if (mapping.getQueue().equals(SECONDARY_GROUP_MAPPING)) {
return getContextForSecondaryGroup(user, mapping); if (LOG.isDebugEnabled()) {
LOG.debug("Creating placement context for user {} using " +
"secondary group mapping", user);
}
return getPlacementContext(mapping, getSecondaryGroup(user));
} else { } else {
if (LOG.isDebugEnabled()) {
LOG.debug("Creating placement context for user {} using " +
"static user static mapping", user);
}
return getPlacementContext(mapping); return getPlacementContext(mapping);
} }
} }
if (user.equals(mapping.getSource())) { if (user.equals(mapping.getSource())) {
if (mapping.getQueue().equals(PRIMARY_GROUP_MAPPING)) { if (mapping.getQueue().equals(PRIMARY_GROUP_MAPPING)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Creating placement context for user {} using " +
"static user primary group mapping", user);
}
return getPlacementContext(mapping, getPrimaryGroup(user)); return getPlacementContext(mapping, getPrimaryGroup(user));
} else if (mapping.getQueue().equals(SECONDARY_GROUP_MAPPING)) { } else if (mapping.getQueue().equals(SECONDARY_GROUP_MAPPING)) {
String secondaryGroup = getSecondaryGroup(user); String secondaryGroup = getSecondaryGroup(user);
if (secondaryGroup != null) { if (secondaryGroup != null) {
if (LOG.isDebugEnabled()) {
LOG.debug("Creating placement context for user {} using " +
"static user secondary group mapping", user);
}
return getPlacementContext(mapping, secondaryGroup); return getPlacementContext(mapping, secondaryGroup);
} else { } else {
if (LOG.isDebugEnabled()) {
LOG.debug("Wanted to create placement context for user {}" +
" using static user secondary group mapping," +
" but user has no secondary group!", user);
}
return null; return null;
} }
} else { } else {
if (LOG.isDebugEnabled()) {
LOG.debug("Creating placement context for user {} using " +
"static user static mapping", user);
}
return getPlacementContext(mapping); return getPlacementContext(mapping);
} }
} }
@ -142,8 +183,16 @@ private ApplicationPlacementContext getPlacementForUser(String user)
for (String userGroups : groups.getGroups(user)) { for (String userGroups : groups.getGroups(user)) {
if (userGroups.equals(mapping.getSource())) { if (userGroups.equals(mapping.getSource())) {
if (mapping.getQueue().equals(CURRENT_USER_MAPPING)) { if (mapping.getQueue().equals(CURRENT_USER_MAPPING)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Creating placement context for user {} using " +
"static group current user mapping", user);
}
return getPlacementContext(mapping, user); return getPlacementContext(mapping, user);
} }
if (LOG.isDebugEnabled()) {
LOG.debug("Creating placement context for user {} using " +
"static group static mapping", user);
}
return getPlacementContext(mapping); return getPlacementContext(mapping);
} }
} }
@ -152,49 +201,23 @@ private ApplicationPlacementContext getPlacementForUser(String user)
return null; return null;
} }
// invoked for mappings: /**
// u:%user:[parent].%primary_group * This convenience method allows to change the parent path or a leafName in
// u:%user:%primary_group * a mapping object, by creating a new one, using the builder and copying the
private ApplicationPlacementContext getContextForPrimaryGroup( * rest of the parameters.
String user, * @param mapping The mapping to be changed
QueueMapping mapping) throws IOException { * @param parentPath The new parentPath of the mapping
String group = * @param leafName The new leafQueueName of the mapping
CapacitySchedulerConfiguration.ROOT + "." + getPrimaryGroup(user); * @return The updated NEW mapping
*/
String parent = mapping.getParentQueue(); private QueueMapping alterMapping(
CSQueue groupQueue = queueManager.getQueue(group); QueueMapping mapping, String parentPath, String leafName) {
return QueueMappingBuilder.create()
if (parent != null) { .type(mapping.getType())
CSQueue parentQueue = queueManager.getQueue(parent); .source(mapping.getSource())
.queue(leafName)
if (parentQueue instanceof ManagedParentQueue) { .parentQueue(parentPath)
return getPlacementContext(mapping, group); .build();
} else {
return groupQueue == null ? null : getPlacementContext(mapping, group);
}
} else {
return groupQueue == null ? null : getPlacementContext(mapping, group);
}
}
// invoked for mappings
// u:%user:%secondary_group
// u:%user:[parent].%secondary_group
private ApplicationPlacementContext getContextForSecondaryGroup(
String user,
QueueMapping mapping) throws IOException {
String secondaryGroup = getSecondaryGroup(user);
if (secondaryGroup != null) {
CSQueue queue = this.queueManager.getQueue(secondaryGroup);
if ( queue != null) {
return getPlacementContext(mapping, queue.getQueuePath());
} else {
return null;
}
} else {
return null;
}
} }
// invoked for mappings: // invoked for mappings:
@ -205,20 +228,24 @@ private ApplicationPlacementContext getContextForGroupParent(
QueueMapping mapping, QueueMapping mapping,
String group) throws IOException { String group) throws IOException {
if (this.queueManager.getQueue(group) != null) { CSQueue groupQueue = this.queueManager.getQueue(group);
if (groupQueue != null) {
// replace the group string // replace the group string
QueueMapping resolvedGroupMapping = QueueMapping resolvedGroupMapping = alterMapping(
QueueMappingBuilder.create() mapping,
.type(mapping.getType()) groupQueue.getQueuePath(),
.source(mapping.getSource()) user);
.queue(user)
.parentQueue(
CapacitySchedulerConfiguration.ROOT + "." +
group)
.build();
validateQueueMapping(resolvedGroupMapping); validateQueueMapping(resolvedGroupMapping);
return getPlacementContext(resolvedGroupMapping, user); return getPlacementContext(resolvedGroupMapping, user);
} else { } else {
if (queueManager.isAmbiguous(group)) {
LOG.info("Queue mapping rule expect group queue to exist with name {}" +
" but the reference is ambiguous!", group);
} else {
LOG.info("Queue mapping rule expect group queue to exist with name {}" +
" but it does not exist!", group);
}
return null; return null;
} }
} }
@ -247,7 +274,7 @@ public ApplicationPlacementContext getPlacementForApp(
} catch (IOException ioex) { } catch (IOException ioex) {
String message = "Failed to submit application " + applicationId + String message = "Failed to submit application " + applicationId +
" submitted by user " + user + " reason: " + ioex.getMessage(); " submitted by user " + user + " reason: " + ioex.getMessage();
throw new YarnException(message); throw new YarnException(message, ioex);
} }
} }
return null; return null;
@ -260,7 +287,6 @@ private ApplicationPlacementContext getPlacementContext(
private ApplicationPlacementContext getPlacementContext(QueueMapping mapping, private ApplicationPlacementContext getPlacementContext(QueueMapping mapping,
String leafQueueName) throws IOException { String leafQueueName) throws IOException {
//leafQueue name no longer identifies a queue uniquely checking ambiguity //leafQueue name no longer identifies a queue uniquely checking ambiguity
if (!mapping.hasParentQueue() && queueManager.isAmbiguous(leafQueueName)) { if (!mapping.hasParentQueue() && queueManager.isAmbiguous(leafQueueName)) {
throw new IOException("mapping contains ambiguous leaf queue reference " + throw new IOException("mapping contains ambiguous leaf queue reference " +
@ -268,13 +294,72 @@ private ApplicationPlacementContext getPlacementContext(QueueMapping mapping,
} }
if (!StringUtils.isEmpty(mapping.getParentQueue())) { if (!StringUtils.isEmpty(mapping.getParentQueue())) {
return new ApplicationPlacementContext(leafQueueName, return getPlacementContextWithParent(mapping, leafQueueName);
mapping.getParentQueue()); } else {
} else{ return getPlacementContextNoParent(leafQueueName);
return new ApplicationPlacementContext(leafQueueName);
} }
} }
private ApplicationPlacementContext getPlacementContextWithParent(
QueueMapping mapping,
String leafQueueName) {
CSQueue parent = queueManager.getQueue(mapping.getParentQueue());
//we don't find the specified parent, so the placement rule is invalid
//for this case
if (parent == null) {
if (queueManager.isAmbiguous(mapping.getParentQueue())) {
LOG.warn("Placement rule specified a parent queue {}, but it is" +
"ambiguous.", mapping.getParentQueue());
} else {
LOG.warn("Placement rule specified a parent queue {}, but it does" +
"not exist.", mapping.getParentQueue());
}
return null;
}
String parentPath = parent.getQueuePath();
//if we have a parent which is not a managed parent, we check if the leaf
//queue exists under this parent
if (!(parent instanceof ManagedParentQueue)) {
CSQueue queue = queueManager.getQueue(
parentPath + "." + leafQueueName);
//if the queue doesn't exit we return null
if (queue == null) {
LOG.warn("Placement rule specified a parent queue {}, but it is" +
" not a managed parent queue, and no queue exists with name {} " +
"under it.", mapping.getParentQueue(), leafQueueName);
return null;
}
}
//at this point we either have a managed parent or the queue actually
//exists so we have a placement context, returning it
return new ApplicationPlacementContext(leafQueueName, parentPath);
}
private ApplicationPlacementContext getPlacementContextNoParent(
String leafQueueName) {
//in this case we don't have a parent specified so we expect the queue to
//exist, otherwise the mapping will not be valid for this case
CSQueue queue = queueManager.getQueue(leafQueueName);
if (queue == null) {
if (queueManager.isAmbiguous(leafQueueName)) {
LOG.warn("Queue {} specified in placement rule is ambiguous",
leafQueueName);
} else {
LOG.warn("Queue {} specified in placement rule does not exist",
leafQueueName);
}
return null;
}
//getting parent path to make sure if the leaf name would become ambiguous
//the placement context stays valid.
CSQueue parent = queueManager.getQueue(leafQueueName).getParent();
return new ApplicationPlacementContext(
leafQueueName, parent.getQueuePath());
}
@VisibleForTesting @VisibleForTesting
@Override @Override
public boolean initialize(ResourceScheduler scheduler) public boolean initialize(ResourceScheduler scheduler)
@ -449,10 +534,12 @@ private void validateQueueMapping(QueueMapping queueMapping)
//as mapping.getQueue() //as mapping.getQueue()
if (leafQueue == null && queueManager.isAmbiguous(leafQueueFullName)) { if (leafQueue == null && queueManager.isAmbiguous(leafQueueFullName)) {
throw new IOException("mapping contains ambiguous leaf queue name: " throw new IOException("mapping contains ambiguous leaf queue name: "
+ leafQueueFullName); + leafQueueFullName);
} else { } else if (parentQueue == null ||
throw new IOException("mapping contains invalid or non-leaf queue : " (!(parentQueue instanceof ManagedParentQueue))) {
+ leafQueueFullName); throw new IOException("mapping contains invalid or non-leaf queue " +
" and no managed parent is found: "
+ leafQueueFullName);
} }
} else if (parentQueue == null || (!(parentQueue instanceof ParentQueue))) { } else if (parentQueue == null || (!(parentQueue instanceof ParentQueue))) {
throw new IOException( throw new IOException(

81
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java
View File

@ -21,6 +21,7 @@
import static org.junit.Assert.fail; import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
import static org.mockito.Mockito.isNull;
import java.util.Arrays; import java.util.Arrays;
@ -70,13 +71,18 @@ private void verifyQueueMapping(QueueMappingTestData queueMappingTestData)
CapacitySchedulerQueueManager queueManager = CapacitySchedulerQueueManager queueManager =
mock(CapacitySchedulerQueueManager.class); mock(CapacitySchedulerQueueManager.class);
ParentQueue root = mock(ParentQueue.class);
when(root.getQueuePath()).thenReturn("root");
ParentQueue agroup = mock(ParentQueue.class); ParentQueue agroup = mock(ParentQueue.class);
when(agroup.getQueuePath()).thenReturn("root.agroup"); when(agroup.getQueuePath()).thenReturn("root.agroup");
ParentQueue bsubgroup2 = mock(ParentQueue.class); ParentQueue bsubgroup2 = mock(ParentQueue.class);
when(bsubgroup2.getQueuePath()).thenReturn("root.bsubgroup2"); when(bsubgroup2.getQueuePath()).thenReturn("root.bsubgroup2");
when(bsubgroup2.getParent()).thenReturn(root);
ManagedParentQueue managedParent = mock(ManagedParentQueue.class); ManagedParentQueue managedParent = mock(ManagedParentQueue.class);
when(managedParent.getQueueName()).thenReturn("root.managedParent"); when(managedParent.getQueueName()).thenReturn("managedParent");
when(managedParent.getQueuePath()).thenReturn("root.managedParent");
LeafQueue a = mock(LeafQueue.class); LeafQueue a = mock(LeafQueue.class);
when(a.getQueuePath()).thenReturn("root.agroup.a"); when(a.getQueuePath()).thenReturn("root.agroup.a");
@ -86,15 +92,23 @@ private void verifyQueueMapping(QueueMappingTestData queueMappingTestData)
when(b.getParent()).thenReturn(bsubgroup2); when(b.getParent()).thenReturn(bsubgroup2);
LeafQueue asubgroup2 = mock(LeafQueue.class); LeafQueue asubgroup2 = mock(LeafQueue.class);
when(asubgroup2.getQueuePath()).thenReturn("root.asubgroup2"); when(asubgroup2.getQueuePath()).thenReturn("root.asubgroup2");
when(asubgroup2.getParent()).thenReturn(root);
when(queueManager.getQueue(isNull())).thenReturn(null);
when(queueManager.getQueue("a")).thenReturn(a); when(queueManager.getQueue("a")).thenReturn(a);
when(a.getParent()).thenReturn(agroup);
when(queueManager.getQueue("b")).thenReturn(b); when(queueManager.getQueue("b")).thenReturn(b);
when(b.getParent()).thenReturn(bsubgroup2);
when(queueManager.getQueue("agroup")).thenReturn(agroup); when(queueManager.getQueue("agroup")).thenReturn(agroup);
when(agroup.getParent()).thenReturn(root);
when(queueManager.getQueue("bsubgroup2")).thenReturn(bsubgroup2); when(queueManager.getQueue("bsubgroup2")).thenReturn(bsubgroup2);
when(bsubgroup2.getParent()).thenReturn(root);
when(queueManager.getQueue("asubgroup2")).thenReturn(asubgroup2); when(queueManager.getQueue("asubgroup2")).thenReturn(asubgroup2);
when(asubgroup2.getParent()).thenReturn(root);
when(queueManager.getQueue("managedParent")).thenReturn(managedParent); when(queueManager.getQueue("managedParent")).thenReturn(managedParent);
when(queueManager.getQueue(null)).thenThrow(new NullPointerException()); when(managedParent.getParent()).thenReturn(root);
when(queueManager.getQueue("root")).thenReturn(root);
when(queueManager.getQueue("root.agroup")).thenReturn(agroup); when(queueManager.getQueue("root.agroup")).thenReturn(agroup);
when(queueManager.getQueue("root.bsubgroup2")).thenReturn(bsubgroup2); when(queueManager.getQueue("root.bsubgroup2")).thenReturn(bsubgroup2);
when(queueManager.getQueue("root.asubgroup2")).thenReturn(asubgroup2); when(queueManager.getQueue("root.asubgroup2")).thenReturn(asubgroup2);
@ -135,7 +149,8 @@ public void testSecondaryGroupMapping() throws YarnException {
.source("%user") .source("%user")
.queue("%secondary_group").build()) .queue("%secondary_group").build())
.inputUser("a") .inputUser("a")
.expectedQueue("root.asubgroup2") .expectedQueue("asubgroup2")
.expectedParentQueue("root")
.build()); .build());
// PrimaryGroupMapping.class returns only primary group, no secondary groups // PrimaryGroupMapping.class returns only primary group, no secondary groups
@ -176,35 +191,37 @@ public void testNullGroupMapping() throws YarnException {
@Test @Test
public void testMapping() throws YarnException { public void testMapping() throws YarnException {
//if a mapping rule definies no parent, we cannot expect auto creation,
// so we must provide already existing queues
verifyQueueMapping( verifyQueueMapping(
QueueMappingTestDataBuilder.create() QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create() .queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER) .type(MappingType.USER)
.source("a") .source("a")
.queue("q1") .queue("a")
.build()) .build())
.inputUser("a") .inputUser("a")
.expectedQueue("q1") .expectedQueue("a")
.build()); .build());
verifyQueueMapping( verifyQueueMapping(
QueueMappingTestDataBuilder.create() QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create() .queueMapping(QueueMappingBuilder.create()
.type(MappingType.GROUP) .type(MappingType.GROUP)
.source("agroup") .source("agroup")
.queue("q1") .queue("a")
.build()) .build())
.inputUser("a") .inputUser("a")
.expectedQueue("q1") .expectedQueue("a")
.build()); .build());
verifyQueueMapping( verifyQueueMapping(
QueueMappingTestDataBuilder.create() QueueMappingTestDataBuilder.create()
.queueMapping(QueueMappingBuilder.create() .queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER) .type(MappingType.USER)
.source("%user") .source("%user")
.queue("q2") .queue("b")
.build()) .build())
.inputUser("a") .inputUser("a")
.expectedQueue("q2") .expectedQueue("b")
.build()); .build());
verifyQueueMapping( verifyQueueMapping(
QueueMappingTestDataBuilder.create() QueueMappingTestDataBuilder.create()
@ -224,7 +241,8 @@ public void testMapping() throws YarnException {
.queue("%primary_group") .queue("%primary_group")
.build()) .build())
.inputUser("a") .inputUser("a")
.expectedQueue("root.agroup") .expectedQueue("agroup")
.expectedParentQueue("root")
.build()); .build());
verifyQueueMapping( verifyQueueMapping(
QueueMappingTestDataBuilder.create() QueueMappingTestDataBuilder.create()
@ -255,10 +273,10 @@ public void testMapping() throws YarnException {
.queueMapping(QueueMappingBuilder.create() .queueMapping(QueueMappingBuilder.create()
.type(MappingType.GROUP) .type(MappingType.GROUP)
.source("asubgroup1") .source("asubgroup1")
.queue("q1") .queue("a")
.build()) .build())
.inputUser("a") .inputUser("a")
.expectedQueue("q1") .expectedQueue("a")
.build()); .build());
// "agroup" queue exists // "agroup" queue exists
@ -268,10 +286,11 @@ public void testMapping() throws YarnException {
.type(MappingType.USER) .type(MappingType.USER)
.source("%user") .source("%user")
.queue("%primary_group") .queue("%primary_group")
.parentQueue("bsubgroup2") .parentQueue("root")
.build()) .build())
.inputUser("a") .inputUser("a")
.expectedQueue("root.agroup") .expectedQueue("agroup")
.expectedParentQueue("root")
.build()); .build());
// "abcgroup" queue doesn't exist, %primary_group queue, not managed parent // "abcgroup" queue doesn't exist, %primary_group queue, not managed parent
@ -297,7 +316,8 @@ public void testMapping() throws YarnException {
.parentQueue("managedParent") .parentQueue("managedParent")
.build()) .build())
.inputUser("abc") .inputUser("abc")
.expectedQueue("root.abcgroup") .expectedQueue("abcgroup")
.expectedParentQueue("root.managedParent")
.build()); .build());
// "abcgroup" queue doesn't exist, %secondary_group queue // "abcgroup" queue doesn't exist, %secondary_group queue
@ -320,10 +340,11 @@ public void testMapping() throws YarnException {
.type(MappingType.USER) .type(MappingType.USER)
.source("%user") .source("%user")
.queue("%secondary_group") .queue("%secondary_group")
.parentQueue("bsubgroup2") .parentQueue("root")
.build()) .build())
.inputUser("a") .inputUser("a")
.expectedQueue("root.asubgroup2") .expectedQueue("asubgroup2")
.expectedParentQueue("root")
.build()); .build());
// specify overwritten, and see if user specified a queue, and it will be // specify overwritten, and see if user specified a queue, and it will be
@ -333,11 +354,11 @@ public void testMapping() throws YarnException {
.queueMapping(QueueMappingBuilder.create() .queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER) .type(MappingType.USER)
.source("user") .source("user")
.queue("q1") .queue("a")
.build()) .build())
.inputUser("user") .inputUser("user")
.inputQueue("q2") .inputQueue("b")
.expectedQueue("q1") .expectedQueue("a")
.overwrite(true) .overwrite(true)
.build()); .build());
@ -347,11 +368,11 @@ public void testMapping() throws YarnException {
.queueMapping(QueueMappingBuilder.create() .queueMapping(QueueMappingBuilder.create()
.type(MappingType.USER) .type(MappingType.USER)
.source("user") .source("user")
.queue("q1") .queue("a")
.build()) .build())
.inputUser("user") .inputUser("user")
.inputQueue("q2") .inputQueue("b")
.expectedQueue("q2") .expectedQueue("b")
.build()); .build());
// if overwritten not specified, it should be which user specified // if overwritten not specified, it should be which user specified
@ -364,8 +385,8 @@ public void testMapping() throws YarnException {
.parentQueue("usergroup") .parentQueue("usergroup")
.build()) .build())
.inputUser("user") .inputUser("user")
.inputQueue("default") .inputQueue("a")
.expectedQueue("user") .expectedQueue("a")
.build()); .build());
// if overwritten not specified, it should be which user specified // if overwritten not specified, it should be which user specified
@ -374,12 +395,12 @@ public void testMapping() throws YarnException {
.queueMapping(QueueMappingBuilder.create() .queueMapping(QueueMappingBuilder.create()
.type(MappingType.GROUP) .type(MappingType.GROUP)
.source("usergroup") .source("usergroup")
.queue("%user") .queue("b")
.parentQueue("usergroup") .parentQueue("root.bsubgroup2")
.build()) .build())
.inputUser("user") .inputUser("user")
.inputQueue("agroup") .inputQueue("a")
.expectedQueue("user") .expectedQueue("b")
.overwrite(true) .overwrite(true)
.build()); .build());
@ -391,7 +412,7 @@ public void testMapping() throws YarnException {
.type(MappingType.GROUP) .type(MappingType.GROUP)
.source("agroup") .source("agroup")
.queue("%user") .queue("%user")
.parentQueue("parent1") .parentQueue("root.agroup")
.build()) .build())
.inputUser("a") .inputUser("a")
.expectedQueue("a") .expectedQueue("a")

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java
View File

@ -206,7 +206,7 @@ public void testNestedUserQueueWithStaticParentQueue() throws Exception {
ApplicationPlacementContext ctx2 = r.getPlacementForApp(asc, "user2"); ApplicationPlacementContext ctx2 = r.getPlacementForApp(asc, "user2");
assertEquals("Queue", "user2", ctx2.getQueue()); assertEquals("Queue", "user2", ctx2.getQueue());
assertEquals("Queue", "c", ctx2.getParentQueue()); assertEquals("Queue", "root.c", ctx2.getParentQueue());
} finally { } finally {
if(mockRM != null) { if(mockRM != null) {
mockRM.close(); mockRM.close();
@ -398,7 +398,7 @@ public void testDynamicPrimaryGroupQueue() throws Exception {
// u:user2:%primary_group // u:user2:%primary_group
QueueMapping userQueueMapping2 = QueueMappingBuilder.create() QueueMapping userQueueMapping2 = QueueMappingBuilder.create()
.type(QueueMapping.MappingType.USER) .type(QueueMapping.MappingType.USER)
.source("user2") .source("a1")
.queue("%primary_group") .queue("%primary_group")
.build(); .build();
@ -430,8 +430,8 @@ public void testDynamicPrimaryGroupQueue() throws Exception {
ApplicationPlacementContext ctx = r.getPlacementForApp(asc, "user1"); ApplicationPlacementContext ctx = r.getPlacementForApp(asc, "user1");
assertEquals("Queue", "b1", ctx.getQueue()); assertEquals("Queue", "b1", ctx.getQueue());
ApplicationPlacementContext ctx1 = r.getPlacementForApp(asc, "user2"); ApplicationPlacementContext ctx1 = r.getPlacementForApp(asc, "a1");
assertEquals("Queue", "user2group", ctx1.getQueue()); assertEquals("Queue", "a1group", ctx1.getQueue());
} finally { } finally {
if (mockRM != null) { if (mockRM != null) {
mockRM.close(); mockRM.close();
@ -467,14 +467,14 @@ public void testFixedUserWithDynamicGroupQueue() throws Exception {
// u:user2:%primary_group // u:user2:%primary_group
QueueMapping userQueueMapping2 = QueueMappingBuilder.create() QueueMapping userQueueMapping2 = QueueMappingBuilder.create()
.type(QueueMapping.MappingType.USER) .type(QueueMapping.MappingType.USER)
.source("user2") .source("a1")
.queue("%primary_group") .queue("%primary_group")
.build(); .build();
// u:b4:%secondary_group // u:b4:%secondary_group
QueueMapping userQueueMapping3 = QueueMappingBuilder.create() QueueMapping userQueueMapping3 = QueueMappingBuilder.create()
.type(QueueMapping.MappingType.USER) .type(QueueMapping.MappingType.USER)
.source("b4") .source("e")
.queue("%secondary_group") .queue("%secondary_group")
.build(); .build();
@ -507,11 +507,11 @@ public void testFixedUserWithDynamicGroupQueue() throws Exception {
ApplicationPlacementContext ctx = r.getPlacementForApp(asc, "user1"); ApplicationPlacementContext ctx = r.getPlacementForApp(asc, "user1");
assertEquals("Queue", "b1", ctx.getQueue()); assertEquals("Queue", "b1", ctx.getQueue());
ApplicationPlacementContext ctx1 = r.getPlacementForApp(asc, "user2"); ApplicationPlacementContext ctx1 = r.getPlacementForApp(asc, "a1");
assertEquals("Queue", "user2group", ctx1.getQueue()); assertEquals("Queue", "a1group", ctx1.getQueue());
ApplicationPlacementContext ctx2 = r.getPlacementForApp(asc, "b4"); ApplicationPlacementContext ctx2 = r.getPlacementForApp(asc, "e");
assertEquals("Queue", "b4subgroup1", ctx2.getQueue()); assertEquals("Queue", "esubgroup1", ctx2.getQueue());
} finally { } finally {
if (mockRM != null) { if (mockRM != null) {
mockRM.close(); mockRM.close();