YARN-10254. CapacityScheduler incorrect User Group Mapping after leaf queue change. Contributed by Gergely Pollak
This commit is contained in:
parent
aa6d13455b
commit
b5efdea4fd
@ -105,35 +105,76 @@ private ApplicationPlacementContext getPlacementForUser(String user)
|
|||||||
if (mapping.getParentQueue() != null
|
if (mapping.getParentQueue() != null
|
||||||
&& mapping.getParentQueue().equals(PRIMARY_GROUP_MAPPING)
|
&& mapping.getParentQueue().equals(PRIMARY_GROUP_MAPPING)
|
||||||
&& mapping.getQueue().equals(CURRENT_USER_MAPPING)) {
|
&& mapping.getQueue().equals(CURRENT_USER_MAPPING)) {
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug("Creating placement context for user {} using " +
|
||||||
|
"primary group current user mapping", user);
|
||||||
|
}
|
||||||
return getContextForGroupParent(user, mapping,
|
return getContextForGroupParent(user, mapping,
|
||||||
getPrimaryGroup(user));
|
getPrimaryGroup(user));
|
||||||
} else if (mapping.getParentQueue() != null
|
} else if (mapping.getParentQueue() != null
|
||||||
&& mapping.getParentQueue().equals(SECONDARY_GROUP_MAPPING)
|
&& mapping.getParentQueue().equals(SECONDARY_GROUP_MAPPING)
|
||||||
&& mapping.getQueue().equals(CURRENT_USER_MAPPING)) {
|
&& mapping.getQueue().equals(CURRENT_USER_MAPPING)) {
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug("Creating placement context for user {} using " +
|
||||||
|
"secondary group current user mapping", user);
|
||||||
|
}
|
||||||
return getContextForGroupParent(user, mapping,
|
return getContextForGroupParent(user, mapping,
|
||||||
getSecondaryGroup(user));
|
getSecondaryGroup(user));
|
||||||
} else if (mapping.getQueue().equals(CURRENT_USER_MAPPING)) {
|
} else if (mapping.getQueue().equals(CURRENT_USER_MAPPING)) {
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug("Creating placement context for user {} using " +
|
||||||
|
"current user mapping", user);
|
||||||
|
}
|
||||||
return getPlacementContext(mapping, user);
|
return getPlacementContext(mapping, user);
|
||||||
} else if (mapping.getQueue().equals(PRIMARY_GROUP_MAPPING)) {
|
} else if (mapping.getQueue().equals(PRIMARY_GROUP_MAPPING)) {
|
||||||
return getContextForPrimaryGroup(user, mapping);
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug("Creating placement context for user {} using " +
|
||||||
|
"primary group mapping", user);
|
||||||
|
}
|
||||||
|
return getPlacementContext(mapping, getPrimaryGroup(user));
|
||||||
} else if (mapping.getQueue().equals(SECONDARY_GROUP_MAPPING)) {
|
} else if (mapping.getQueue().equals(SECONDARY_GROUP_MAPPING)) {
|
||||||
return getContextForSecondaryGroup(user, mapping);
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug("Creating placement context for user {} using " +
|
||||||
|
"secondary group mapping", user);
|
||||||
|
}
|
||||||
|
return getPlacementContext(mapping, getSecondaryGroup(user));
|
||||||
} else {
|
} else {
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug("Creating placement context for user {} using " +
|
||||||
|
"static user static mapping", user);
|
||||||
|
}
|
||||||
return getPlacementContext(mapping);
|
return getPlacementContext(mapping);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (user.equals(mapping.getSource())) {
|
if (user.equals(mapping.getSource())) {
|
||||||
if (mapping.getQueue().equals(PRIMARY_GROUP_MAPPING)) {
|
if (mapping.getQueue().equals(PRIMARY_GROUP_MAPPING)) {
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug("Creating placement context for user {} using " +
|
||||||
|
"static user primary group mapping", user);
|
||||||
|
}
|
||||||
return getPlacementContext(mapping, getPrimaryGroup(user));
|
return getPlacementContext(mapping, getPrimaryGroup(user));
|
||||||
} else if (mapping.getQueue().equals(SECONDARY_GROUP_MAPPING)) {
|
} else if (mapping.getQueue().equals(SECONDARY_GROUP_MAPPING)) {
|
||||||
String secondaryGroup = getSecondaryGroup(user);
|
String secondaryGroup = getSecondaryGroup(user);
|
||||||
if (secondaryGroup != null) {
|
if (secondaryGroup != null) {
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug("Creating placement context for user {} using " +
|
||||||
|
"static user secondary group mapping", user);
|
||||||
|
}
|
||||||
return getPlacementContext(mapping, secondaryGroup);
|
return getPlacementContext(mapping, secondaryGroup);
|
||||||
} else {
|
} else {
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug("Wanted to create placement context for user {}" +
|
||||||
|
" using static user secondary group mapping," +
|
||||||
|
" but user has no secondary group!", user);
|
||||||
|
}
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug("Creating placement context for user {} using " +
|
||||||
|
"static user static mapping", user);
|
||||||
|
}
|
||||||
return getPlacementContext(mapping);
|
return getPlacementContext(mapping);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -142,8 +183,16 @@ private ApplicationPlacementContext getPlacementForUser(String user)
|
|||||||
for (String userGroups : groups.getGroups(user)) {
|
for (String userGroups : groups.getGroups(user)) {
|
||||||
if (userGroups.equals(mapping.getSource())) {
|
if (userGroups.equals(mapping.getSource())) {
|
||||||
if (mapping.getQueue().equals(CURRENT_USER_MAPPING)) {
|
if (mapping.getQueue().equals(CURRENT_USER_MAPPING)) {
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug("Creating placement context for user {} using " +
|
||||||
|
"static group current user mapping", user);
|
||||||
|
}
|
||||||
return getPlacementContext(mapping, user);
|
return getPlacementContext(mapping, user);
|
||||||
}
|
}
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug("Creating placement context for user {} using " +
|
||||||
|
"static group static mapping", user);
|
||||||
|
}
|
||||||
return getPlacementContext(mapping);
|
return getPlacementContext(mapping);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -152,49 +201,23 @@ private ApplicationPlacementContext getPlacementForUser(String user)
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
// invoked for mappings:
|
/**
|
||||||
// u:%user:[parent].%primary_group
|
* This convenience method allows to change the parent path or a leafName in
|
||||||
// u:%user:%primary_group
|
* a mapping object, by creating a new one, using the builder and copying the
|
||||||
private ApplicationPlacementContext getContextForPrimaryGroup(
|
* rest of the parameters.
|
||||||
String user,
|
* @param mapping The mapping to be changed
|
||||||
QueueMapping mapping) throws IOException {
|
* @param parentPath The new parentPath of the mapping
|
||||||
String group =
|
* @param leafName The new leafQueueName of the mapping
|
||||||
CapacitySchedulerConfiguration.ROOT + "." + getPrimaryGroup(user);
|
* @return The updated NEW mapping
|
||||||
|
*/
|
||||||
String parent = mapping.getParentQueue();
|
private QueueMapping alterMapping(
|
||||||
CSQueue groupQueue = queueManager.getQueue(group);
|
QueueMapping mapping, String parentPath, String leafName) {
|
||||||
|
return QueueMappingBuilder.create()
|
||||||
if (parent != null) {
|
.type(mapping.getType())
|
||||||
CSQueue parentQueue = queueManager.getQueue(parent);
|
.source(mapping.getSource())
|
||||||
|
.queue(leafName)
|
||||||
if (parentQueue instanceof ManagedParentQueue) {
|
.parentQueue(parentPath)
|
||||||
return getPlacementContext(mapping, group);
|
.build();
|
||||||
} else {
|
|
||||||
return groupQueue == null ? null : getPlacementContext(mapping, group);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
return groupQueue == null ? null : getPlacementContext(mapping, group);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// invoked for mappings
|
|
||||||
// u:%user:%secondary_group
|
|
||||||
// u:%user:[parent].%secondary_group
|
|
||||||
private ApplicationPlacementContext getContextForSecondaryGroup(
|
|
||||||
String user,
|
|
||||||
QueueMapping mapping) throws IOException {
|
|
||||||
String secondaryGroup = getSecondaryGroup(user);
|
|
||||||
|
|
||||||
if (secondaryGroup != null) {
|
|
||||||
CSQueue queue = this.queueManager.getQueue(secondaryGroup);
|
|
||||||
if ( queue != null) {
|
|
||||||
return getPlacementContext(mapping, queue.getQueuePath());
|
|
||||||
} else {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// invoked for mappings:
|
// invoked for mappings:
|
||||||
@ -205,20 +228,23 @@ private ApplicationPlacementContext getContextForGroupParent(
|
|||||||
QueueMapping mapping,
|
QueueMapping mapping,
|
||||||
String group) throws IOException {
|
String group) throws IOException {
|
||||||
|
|
||||||
if (this.queueManager.getQueue(group) != null) {
|
CSQueue groupQueue = this.queueManager.getQueue(group);
|
||||||
|
if (groupQueue != null) {
|
||||||
// replace the group string
|
// replace the group string
|
||||||
QueueMapping resolvedGroupMapping =
|
QueueMapping resolvedGroupMapping = alterMapping(
|
||||||
QueueMappingBuilder.create()
|
mapping,
|
||||||
.type(mapping.getType())
|
groupQueue.getQueuePath(),
|
||||||
.source(mapping.getSource())
|
user);
|
||||||
.queue(user)
|
|
||||||
.parentQueue(
|
|
||||||
CapacitySchedulerConfiguration.ROOT + "." +
|
|
||||||
group)
|
|
||||||
.build();
|
|
||||||
validateQueueMapping(resolvedGroupMapping);
|
validateQueueMapping(resolvedGroupMapping);
|
||||||
return getPlacementContext(resolvedGroupMapping, user);
|
return getPlacementContext(resolvedGroupMapping, user);
|
||||||
} else {
|
} else {
|
||||||
|
if (queueManager.isAmbiguous(group)) {
|
||||||
|
LOG.info("Queue mapping rule expect group queue to exist with name {}" +
|
||||||
|
" but the reference is ambiguous!", group);
|
||||||
|
} else {
|
||||||
|
LOG.info("Queue mapping rule expect group queue to exist with name {}" +
|
||||||
|
" but it does not exist!", group);
|
||||||
|
}
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -247,7 +273,7 @@ public ApplicationPlacementContext getPlacementForApp(
|
|||||||
} catch (IOException ioex) {
|
} catch (IOException ioex) {
|
||||||
String message = "Failed to submit application " + applicationId +
|
String message = "Failed to submit application " + applicationId +
|
||||||
" submitted by user " + user + " reason: " + ioex.getMessage();
|
" submitted by user " + user + " reason: " + ioex.getMessage();
|
||||||
throw new YarnException(message);
|
throw new YarnException(message, ioex);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
@ -260,7 +286,6 @@ private ApplicationPlacementContext getPlacementContext(
|
|||||||
|
|
||||||
private ApplicationPlacementContext getPlacementContext(QueueMapping mapping,
|
private ApplicationPlacementContext getPlacementContext(QueueMapping mapping,
|
||||||
String leafQueueName) throws IOException {
|
String leafQueueName) throws IOException {
|
||||||
|
|
||||||
//leafQueue name no longer identifies a queue uniquely checking ambiguity
|
//leafQueue name no longer identifies a queue uniquely checking ambiguity
|
||||||
if (!mapping.hasParentQueue() && queueManager.isAmbiguous(leafQueueName)) {
|
if (!mapping.hasParentQueue() && queueManager.isAmbiguous(leafQueueName)) {
|
||||||
throw new IOException("mapping contains ambiguous leaf queue reference " +
|
throw new IOException("mapping contains ambiguous leaf queue reference " +
|
||||||
@ -268,13 +293,71 @@ private ApplicationPlacementContext getPlacementContext(QueueMapping mapping,
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (!StringUtils.isEmpty(mapping.getParentQueue())) {
|
if (!StringUtils.isEmpty(mapping.getParentQueue())) {
|
||||||
return new ApplicationPlacementContext(leafQueueName,
|
return getPlacementContextWithParent(mapping, leafQueueName);
|
||||||
mapping.getParentQueue());
|
} else {
|
||||||
} else{
|
return getPlacementContextNoParent(leafQueueName);
|
||||||
return new ApplicationPlacementContext(leafQueueName);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private ApplicationPlacementContext getPlacementContextWithParent(
|
||||||
|
QueueMapping mapping,
|
||||||
|
String leafQueueName) {
|
||||||
|
CSQueue parent = queueManager.getQueue(mapping.getParentQueue());
|
||||||
|
//we don't find the specified parent, so the placement rule is invalid
|
||||||
|
//for this case
|
||||||
|
if (parent == null) {
|
||||||
|
if (queueManager.isAmbiguous(mapping.getParentQueue())) {
|
||||||
|
LOG.warn("Placement rule specified a parent queue {}, but it is" +
|
||||||
|
"ambiguous.", mapping.getParentQueue());
|
||||||
|
} else {
|
||||||
|
LOG.warn("Placement rule specified a parent queue {}, but it does" +
|
||||||
|
"not exist.", mapping.getParentQueue());
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
String parentPath = parent.getQueuePath();
|
||||||
|
|
||||||
|
//if we have a parent which is not a managed parent, we check if the leaf
|
||||||
|
//queue exists under this parent
|
||||||
|
if (!(parent instanceof ManagedParentQueue)) {
|
||||||
|
CSQueue queue = queueManager.getQueue(parentPath + "." + leafQueueName);
|
||||||
|
//if the queue doesn't exit we return null
|
||||||
|
if (queue == null) {
|
||||||
|
LOG.warn("Placement rule specified a parent queue {}, but it is" +
|
||||||
|
" not a managed parent queue, and no queue exists with name {} " +
|
||||||
|
"under it.", mapping.getParentQueue(), leafQueueName);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
//at this point we either have a managed parent or the queue actually
|
||||||
|
//exists so we have a placement context, returning it
|
||||||
|
return new ApplicationPlacementContext(leafQueueName, parentPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
private ApplicationPlacementContext getPlacementContextNoParent(
|
||||||
|
String leafQueueName) {
|
||||||
|
//in this case we don't have a parent specified so we expect the queue to
|
||||||
|
//exist, otherwise the mapping will not be valid for this case
|
||||||
|
CSQueue queue = queueManager.getQueue(leafQueueName);
|
||||||
|
if (queue == null) {
|
||||||
|
if (queueManager.isAmbiguous(leafQueueName)) {
|
||||||
|
LOG.warn("Queue {} specified in placement rule is ambiguous",
|
||||||
|
leafQueueName);
|
||||||
|
} else {
|
||||||
|
LOG.warn("Queue {} specified in placement rule does not exist",
|
||||||
|
leafQueueName);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
//getting parent path to make sure if the leaf name would become ambiguous
|
||||||
|
//the placement context stays valid.
|
||||||
|
CSQueue parent = queueManager.getQueue(leafQueueName).getParent();
|
||||||
|
return new ApplicationPlacementContext(
|
||||||
|
leafQueueName, parent.getQueuePath());
|
||||||
|
}
|
||||||
|
|
||||||
@VisibleForTesting
|
@VisibleForTesting
|
||||||
@Override
|
@Override
|
||||||
public boolean initialize(ResourceScheduler scheduler)
|
public boolean initialize(ResourceScheduler scheduler)
|
||||||
@ -449,10 +532,12 @@ private void validateQueueMapping(QueueMapping queueMapping)
|
|||||||
//as mapping.getQueue()
|
//as mapping.getQueue()
|
||||||
if (leafQueue == null && queueManager.isAmbiguous(leafQueueFullName)) {
|
if (leafQueue == null && queueManager.isAmbiguous(leafQueueFullName)) {
|
||||||
throw new IOException("mapping contains ambiguous leaf queue name: "
|
throw new IOException("mapping contains ambiguous leaf queue name: "
|
||||||
+ leafQueueFullName);
|
+ leafQueueFullName);
|
||||||
} else {
|
} else if (parentQueue == null ||
|
||||||
throw new IOException("mapping contains invalid or non-leaf queue : "
|
(!(parentQueue instanceof ManagedParentQueue))) {
|
||||||
+ leafQueueFullName);
|
throw new IOException("mapping contains invalid or non-leaf queue " +
|
||||||
|
" and no managed parent is found: "
|
||||||
|
+ leafQueueFullName);
|
||||||
}
|
}
|
||||||
} else if (parentQueue == null || (!(parentQueue instanceof ParentQueue))) {
|
} else if (parentQueue == null || (!(parentQueue instanceof ParentQueue))) {
|
||||||
throw new IOException(
|
throw new IOException(
|
||||||
|
@ -21,6 +21,7 @@
|
|||||||
import static org.junit.Assert.fail;
|
import static org.junit.Assert.fail;
|
||||||
import static org.mockito.Mockito.mock;
|
import static org.mockito.Mockito.mock;
|
||||||
import static org.mockito.Mockito.when;
|
import static org.mockito.Mockito.when;
|
||||||
|
import static org.mockito.Mockito.isNull;
|
||||||
|
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
|
|
||||||
@ -70,13 +71,18 @@ private void verifyQueueMapping(QueueMappingTestData queueMappingTestData)
|
|||||||
CapacitySchedulerQueueManager queueManager =
|
CapacitySchedulerQueueManager queueManager =
|
||||||
mock(CapacitySchedulerQueueManager.class);
|
mock(CapacitySchedulerQueueManager.class);
|
||||||
|
|
||||||
|
ParentQueue root = mock(ParentQueue.class);
|
||||||
|
when(root.getQueuePath()).thenReturn("root");
|
||||||
|
|
||||||
ParentQueue agroup = mock(ParentQueue.class);
|
ParentQueue agroup = mock(ParentQueue.class);
|
||||||
when(agroup.getQueuePath()).thenReturn("root.agroup");
|
when(agroup.getQueuePath()).thenReturn("root.agroup");
|
||||||
ParentQueue bsubgroup2 = mock(ParentQueue.class);
|
ParentQueue bsubgroup2 = mock(ParentQueue.class);
|
||||||
when(bsubgroup2.getQueuePath()).thenReturn("root.bsubgroup2");
|
when(bsubgroup2.getQueuePath()).thenReturn("root.bsubgroup2");
|
||||||
|
when(bsubgroup2.getParent()).thenReturn(root);
|
||||||
|
|
||||||
ManagedParentQueue managedParent = mock(ManagedParentQueue.class);
|
ManagedParentQueue managedParent = mock(ManagedParentQueue.class);
|
||||||
when(managedParent.getQueueName()).thenReturn("root.managedParent");
|
when(managedParent.getQueueName()).thenReturn("managedParent");
|
||||||
|
when(managedParent.getQueuePath()).thenReturn("root.managedParent");
|
||||||
|
|
||||||
LeafQueue a = mock(LeafQueue.class);
|
LeafQueue a = mock(LeafQueue.class);
|
||||||
when(a.getQueuePath()).thenReturn("root.agroup.a");
|
when(a.getQueuePath()).thenReturn("root.agroup.a");
|
||||||
@ -86,15 +92,23 @@ private void verifyQueueMapping(QueueMappingTestData queueMappingTestData)
|
|||||||
when(b.getParent()).thenReturn(bsubgroup2);
|
when(b.getParent()).thenReturn(bsubgroup2);
|
||||||
LeafQueue asubgroup2 = mock(LeafQueue.class);
|
LeafQueue asubgroup2 = mock(LeafQueue.class);
|
||||||
when(asubgroup2.getQueuePath()).thenReturn("root.asubgroup2");
|
when(asubgroup2.getQueuePath()).thenReturn("root.asubgroup2");
|
||||||
|
when(asubgroup2.getParent()).thenReturn(root);
|
||||||
|
|
||||||
|
when(queueManager.getQueue(isNull())).thenReturn(null);
|
||||||
when(queueManager.getQueue("a")).thenReturn(a);
|
when(queueManager.getQueue("a")).thenReturn(a);
|
||||||
|
when(a.getParent()).thenReturn(agroup);
|
||||||
when(queueManager.getQueue("b")).thenReturn(b);
|
when(queueManager.getQueue("b")).thenReturn(b);
|
||||||
|
when(b.getParent()).thenReturn(bsubgroup2);
|
||||||
when(queueManager.getQueue("agroup")).thenReturn(agroup);
|
when(queueManager.getQueue("agroup")).thenReturn(agroup);
|
||||||
|
when(agroup.getParent()).thenReturn(root);
|
||||||
when(queueManager.getQueue("bsubgroup2")).thenReturn(bsubgroup2);
|
when(queueManager.getQueue("bsubgroup2")).thenReturn(bsubgroup2);
|
||||||
|
when(bsubgroup2.getParent()).thenReturn(root);
|
||||||
when(queueManager.getQueue("asubgroup2")).thenReturn(asubgroup2);
|
when(queueManager.getQueue("asubgroup2")).thenReturn(asubgroup2);
|
||||||
|
when(asubgroup2.getParent()).thenReturn(root);
|
||||||
when(queueManager.getQueue("managedParent")).thenReturn(managedParent);
|
when(queueManager.getQueue("managedParent")).thenReturn(managedParent);
|
||||||
when(queueManager.getQueue(null)).thenThrow(new NullPointerException());
|
when(managedParent.getParent()).thenReturn(root);
|
||||||
|
|
||||||
|
when(queueManager.getQueue("root")).thenReturn(root);
|
||||||
when(queueManager.getQueue("root.agroup")).thenReturn(agroup);
|
when(queueManager.getQueue("root.agroup")).thenReturn(agroup);
|
||||||
when(queueManager.getQueue("root.bsubgroup2")).thenReturn(bsubgroup2);
|
when(queueManager.getQueue("root.bsubgroup2")).thenReturn(bsubgroup2);
|
||||||
when(queueManager.getQueue("root.asubgroup2")).thenReturn(asubgroup2);
|
when(queueManager.getQueue("root.asubgroup2")).thenReturn(asubgroup2);
|
||||||
@ -135,7 +149,8 @@ public void testSecondaryGroupMapping() throws YarnException {
|
|||||||
.source("%user")
|
.source("%user")
|
||||||
.queue("%secondary_group").build())
|
.queue("%secondary_group").build())
|
||||||
.inputUser("a")
|
.inputUser("a")
|
||||||
.expectedQueue("root.asubgroup2")
|
.expectedQueue("asubgroup2")
|
||||||
|
.expectedParentQueue("root")
|
||||||
.build());
|
.build());
|
||||||
|
|
||||||
// PrimaryGroupMapping.class returns only primary group, no secondary groups
|
// PrimaryGroupMapping.class returns only primary group, no secondary groups
|
||||||
@ -176,35 +191,37 @@ public void testNullGroupMapping() throws YarnException {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testMapping() throws YarnException {
|
public void testMapping() throws YarnException {
|
||||||
|
//if a mapping rule defines no parent, we cannot expect auto creation,
|
||||||
|
// so we must provide already existing queues
|
||||||
verifyQueueMapping(
|
verifyQueueMapping(
|
||||||
QueueMappingTestDataBuilder.create()
|
QueueMappingTestDataBuilder.create()
|
||||||
.queueMapping(QueueMappingBuilder.create()
|
.queueMapping(QueueMappingBuilder.create()
|
||||||
.type(MappingType.USER)
|
.type(MappingType.USER)
|
||||||
.source("a")
|
.source("a")
|
||||||
.queue("q1")
|
.queue("a")
|
||||||
.build())
|
.build())
|
||||||
.inputUser("a")
|
.inputUser("a")
|
||||||
.expectedQueue("q1")
|
.expectedQueue("a")
|
||||||
.build());
|
.build());
|
||||||
verifyQueueMapping(
|
verifyQueueMapping(
|
||||||
QueueMappingTestDataBuilder.create()
|
QueueMappingTestDataBuilder.create()
|
||||||
.queueMapping(QueueMappingBuilder.create()
|
.queueMapping(QueueMappingBuilder.create()
|
||||||
.type(MappingType.GROUP)
|
.type(MappingType.GROUP)
|
||||||
.source("agroup")
|
.source("agroup")
|
||||||
.queue("q1")
|
.queue("a")
|
||||||
.build())
|
.build())
|
||||||
.inputUser("a")
|
.inputUser("a")
|
||||||
.expectedQueue("q1")
|
.expectedQueue("a")
|
||||||
.build());
|
.build());
|
||||||
verifyQueueMapping(
|
verifyQueueMapping(
|
||||||
QueueMappingTestDataBuilder.create()
|
QueueMappingTestDataBuilder.create()
|
||||||
.queueMapping(QueueMappingBuilder.create()
|
.queueMapping(QueueMappingBuilder.create()
|
||||||
.type(MappingType.USER)
|
.type(MappingType.USER)
|
||||||
.source("%user")
|
.source("%user")
|
||||||
.queue("q2")
|
.queue("b")
|
||||||
.build())
|
.build())
|
||||||
.inputUser("a")
|
.inputUser("a")
|
||||||
.expectedQueue("q2")
|
.expectedQueue("b")
|
||||||
.build());
|
.build());
|
||||||
verifyQueueMapping(
|
verifyQueueMapping(
|
||||||
QueueMappingTestDataBuilder.create()
|
QueueMappingTestDataBuilder.create()
|
||||||
@ -224,7 +241,8 @@ public void testMapping() throws YarnException {
|
|||||||
.queue("%primary_group")
|
.queue("%primary_group")
|
||||||
.build())
|
.build())
|
||||||
.inputUser("a")
|
.inputUser("a")
|
||||||
.expectedQueue("root.agroup")
|
.expectedQueue("agroup")
|
||||||
|
.expectedParentQueue("root")
|
||||||
.build());
|
.build());
|
||||||
verifyQueueMapping(
|
verifyQueueMapping(
|
||||||
QueueMappingTestDataBuilder.create()
|
QueueMappingTestDataBuilder.create()
|
||||||
@ -255,10 +273,10 @@ public void testMapping() throws YarnException {
|
|||||||
.queueMapping(QueueMappingBuilder.create()
|
.queueMapping(QueueMappingBuilder.create()
|
||||||
.type(MappingType.GROUP)
|
.type(MappingType.GROUP)
|
||||||
.source("asubgroup1")
|
.source("asubgroup1")
|
||||||
.queue("q1")
|
.queue("a")
|
||||||
.build())
|
.build())
|
||||||
.inputUser("a")
|
.inputUser("a")
|
||||||
.expectedQueue("q1")
|
.expectedQueue("a")
|
||||||
.build());
|
.build());
|
||||||
|
|
||||||
// "agroup" queue exists
|
// "agroup" queue exists
|
||||||
@ -268,10 +286,11 @@ public void testMapping() throws YarnException {
|
|||||||
.type(MappingType.USER)
|
.type(MappingType.USER)
|
||||||
.source("%user")
|
.source("%user")
|
||||||
.queue("%primary_group")
|
.queue("%primary_group")
|
||||||
.parentQueue("bsubgroup2")
|
.parentQueue("root")
|
||||||
.build())
|
.build())
|
||||||
.inputUser("a")
|
.inputUser("a")
|
||||||
.expectedQueue("root.agroup")
|
.expectedQueue("agroup")
|
||||||
|
.expectedParentQueue("root")
|
||||||
.build());
|
.build());
|
||||||
|
|
||||||
// "abcgroup" queue doesn't exist, %primary_group queue, not managed parent
|
// "abcgroup" queue doesn't exist, %primary_group queue, not managed parent
|
||||||
@ -297,7 +316,8 @@ public void testMapping() throws YarnException {
|
|||||||
.parentQueue("managedParent")
|
.parentQueue("managedParent")
|
||||||
.build())
|
.build())
|
||||||
.inputUser("abc")
|
.inputUser("abc")
|
||||||
.expectedQueue("root.abcgroup")
|
.expectedQueue("abcgroup")
|
||||||
|
.expectedParentQueue("root.managedParent")
|
||||||
.build());
|
.build());
|
||||||
|
|
||||||
// "abcgroup" queue doesn't exist, %secondary_group queue
|
// "abcgroup" queue doesn't exist, %secondary_group queue
|
||||||
@ -320,10 +340,11 @@ public void testMapping() throws YarnException {
|
|||||||
.type(MappingType.USER)
|
.type(MappingType.USER)
|
||||||
.source("%user")
|
.source("%user")
|
||||||
.queue("%secondary_group")
|
.queue("%secondary_group")
|
||||||
.parentQueue("bsubgroup2")
|
.parentQueue("root")
|
||||||
.build())
|
.build())
|
||||||
.inputUser("a")
|
.inputUser("a")
|
||||||
.expectedQueue("root.asubgroup2")
|
.expectedQueue("asubgroup2")
|
||||||
|
.expectedParentQueue("root")
|
||||||
.build());
|
.build());
|
||||||
|
|
||||||
// specify overwritten, and see if user specified a queue, and it will be
|
// specify overwritten, and see if user specified a queue, and it will be
|
||||||
@ -333,11 +354,11 @@ public void testMapping() throws YarnException {
|
|||||||
.queueMapping(QueueMappingBuilder.create()
|
.queueMapping(QueueMappingBuilder.create()
|
||||||
.type(MappingType.USER)
|
.type(MappingType.USER)
|
||||||
.source("user")
|
.source("user")
|
||||||
.queue("q1")
|
.queue("a")
|
||||||
.build())
|
.build())
|
||||||
.inputUser("user")
|
.inputUser("user")
|
||||||
.inputQueue("q2")
|
.inputQueue("b")
|
||||||
.expectedQueue("q1")
|
.expectedQueue("a")
|
||||||
.overwrite(true)
|
.overwrite(true)
|
||||||
.build());
|
.build());
|
||||||
|
|
||||||
@ -347,11 +368,11 @@ public void testMapping() throws YarnException {
|
|||||||
.queueMapping(QueueMappingBuilder.create()
|
.queueMapping(QueueMappingBuilder.create()
|
||||||
.type(MappingType.USER)
|
.type(MappingType.USER)
|
||||||
.source("user")
|
.source("user")
|
||||||
.queue("q1")
|
.queue("a")
|
||||||
.build())
|
.build())
|
||||||
.inputUser("user")
|
.inputUser("user")
|
||||||
.inputQueue("q2")
|
.inputQueue("b")
|
||||||
.expectedQueue("q2")
|
.expectedQueue("b")
|
||||||
.build());
|
.build());
|
||||||
|
|
||||||
// if overwritten not specified, it should be which user specified
|
// if overwritten not specified, it should be which user specified
|
||||||
@ -364,8 +385,8 @@ public void testMapping() throws YarnException {
|
|||||||
.parentQueue("usergroup")
|
.parentQueue("usergroup")
|
||||||
.build())
|
.build())
|
||||||
.inputUser("user")
|
.inputUser("user")
|
||||||
.inputQueue("default")
|
.inputQueue("a")
|
||||||
.expectedQueue("user")
|
.expectedQueue("a")
|
||||||
.build());
|
.build());
|
||||||
|
|
||||||
// if overwritten not specified, it should be which user specified
|
// if overwritten not specified, it should be which user specified
|
||||||
@ -374,12 +395,12 @@ public void testMapping() throws YarnException {
|
|||||||
.queueMapping(QueueMappingBuilder.create()
|
.queueMapping(QueueMappingBuilder.create()
|
||||||
.type(MappingType.GROUP)
|
.type(MappingType.GROUP)
|
||||||
.source("usergroup")
|
.source("usergroup")
|
||||||
.queue("%user")
|
.queue("b")
|
||||||
.parentQueue("usergroup")
|
.parentQueue("root.bsubgroup2")
|
||||||
.build())
|
.build())
|
||||||
.inputUser("user")
|
.inputUser("user")
|
||||||
.inputQueue("agroup")
|
.inputQueue("a")
|
||||||
.expectedQueue("user")
|
.expectedQueue("b")
|
||||||
.overwrite(true)
|
.overwrite(true)
|
||||||
.build());
|
.build());
|
||||||
|
|
||||||
@ -391,7 +412,7 @@ public void testMapping() throws YarnException {
|
|||||||
.type(MappingType.GROUP)
|
.type(MappingType.GROUP)
|
||||||
.source("agroup")
|
.source("agroup")
|
||||||
.queue("%user")
|
.queue("%user")
|
||||||
.parentQueue("parent1")
|
.parentQueue("root.agroup")
|
||||||
.build())
|
.build())
|
||||||
.inputUser("a")
|
.inputUser("a")
|
||||||
.expectedQueue("a")
|
.expectedQueue("a")
|
||||||
|
@ -206,7 +206,7 @@ public void testNestedUserQueueWithStaticParentQueue() throws Exception {
|
|||||||
|
|
||||||
ApplicationPlacementContext ctx2 = r.getPlacementForApp(asc, "user2");
|
ApplicationPlacementContext ctx2 = r.getPlacementForApp(asc, "user2");
|
||||||
assertEquals("Queue", "user2", ctx2.getQueue());
|
assertEquals("Queue", "user2", ctx2.getQueue());
|
||||||
assertEquals("Queue", "c", ctx2.getParentQueue());
|
assertEquals("Queue", "root.c", ctx2.getParentQueue());
|
||||||
} finally {
|
} finally {
|
||||||
if(mockRM != null) {
|
if(mockRM != null) {
|
||||||
mockRM.close();
|
mockRM.close();
|
||||||
@ -398,7 +398,7 @@ public void testDynamicPrimaryGroupQueue() throws Exception {
|
|||||||
// u:user2:%primary_group
|
// u:user2:%primary_group
|
||||||
QueueMapping userQueueMapping2 = QueueMappingBuilder.create()
|
QueueMapping userQueueMapping2 = QueueMappingBuilder.create()
|
||||||
.type(QueueMapping.MappingType.USER)
|
.type(QueueMapping.MappingType.USER)
|
||||||
.source("user2")
|
.source("a1")
|
||||||
.queue("%primary_group")
|
.queue("%primary_group")
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
@ -430,8 +430,8 @@ public void testDynamicPrimaryGroupQueue() throws Exception {
|
|||||||
ApplicationPlacementContext ctx = r.getPlacementForApp(asc, "user1");
|
ApplicationPlacementContext ctx = r.getPlacementForApp(asc, "user1");
|
||||||
assertEquals("Queue", "b1", ctx.getQueue());
|
assertEquals("Queue", "b1", ctx.getQueue());
|
||||||
|
|
||||||
ApplicationPlacementContext ctx1 = r.getPlacementForApp(asc, "user2");
|
ApplicationPlacementContext ctx1 = r.getPlacementForApp(asc, "a1");
|
||||||
assertEquals("Queue", "user2group", ctx1.getQueue());
|
assertEquals("Queue", "a1group", ctx1.getQueue());
|
||||||
} finally {
|
} finally {
|
||||||
if (mockRM != null) {
|
if (mockRM != null) {
|
||||||
mockRM.close();
|
mockRM.close();
|
||||||
@ -467,14 +467,14 @@ public void testFixedUserWithDynamicGroupQueue() throws Exception {
|
|||||||
// u:user2:%primary_group
|
// u:user2:%primary_group
|
||||||
QueueMapping userQueueMapping2 = QueueMappingBuilder.create()
|
QueueMapping userQueueMapping2 = QueueMappingBuilder.create()
|
||||||
.type(QueueMapping.MappingType.USER)
|
.type(QueueMapping.MappingType.USER)
|
||||||
.source("user2")
|
.source("a1")
|
||||||
.queue("%primary_group")
|
.queue("%primary_group")
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
// u:b4:%secondary_group
|
// u:b4:%secondary_group
|
||||||
QueueMapping userQueueMapping3 = QueueMappingBuilder.create()
|
QueueMapping userQueueMapping3 = QueueMappingBuilder.create()
|
||||||
.type(QueueMapping.MappingType.USER)
|
.type(QueueMapping.MappingType.USER)
|
||||||
.source("b4")
|
.source("e")
|
||||||
.queue("%secondary_group")
|
.queue("%secondary_group")
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
@ -507,11 +507,11 @@ public void testFixedUserWithDynamicGroupQueue() throws Exception {
|
|||||||
ApplicationPlacementContext ctx = r.getPlacementForApp(asc, "user1");
|
ApplicationPlacementContext ctx = r.getPlacementForApp(asc, "user1");
|
||||||
assertEquals("Queue", "b1", ctx.getQueue());
|
assertEquals("Queue", "b1", ctx.getQueue());
|
||||||
|
|
||||||
ApplicationPlacementContext ctx1 = r.getPlacementForApp(asc, "user2");
|
ApplicationPlacementContext ctx1 = r.getPlacementForApp(asc, "a1");
|
||||||
assertEquals("Queue", "user2group", ctx1.getQueue());
|
assertEquals("Queue", "a1group", ctx1.getQueue());
|
||||||
|
|
||||||
ApplicationPlacementContext ctx2 = r.getPlacementForApp(asc, "b4");
|
ApplicationPlacementContext ctx2 = r.getPlacementForApp(asc, "e");
|
||||||
assertEquals("Queue", "b4subgroup1", ctx2.getQueue());
|
assertEquals("Queue", "esubgroup1", ctx2.getQueue());
|
||||||
} finally {
|
} finally {
|
||||||
if (mockRM != null) {
|
if (mockRM != null) {
|
||||||
mockRM.close();
|
mockRM.close();
|
||||||
|
Loading…
Reference in New Issue
Block a user