big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-9868. Validate %primary_group queue in CS queue manager. Contributed by Manikandan R

Browse Source
This commit is contained in:
Szilard Nemeth 2020-01-13 17:15:09 +01:00
parent 2576c31644
commit b7ef8a333f
4 changed files with 41 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
View File

@ -179,11 +179,15 @@ private ApplicationPlacementContext getPlacementForUser(String user)
if (mapping.getParentQueue() != null
&& mapping.getParentQueue().equals(PRIMARY_GROUP_MAPPING)
&& mapping.getQueue().equals(CURRENT_USER_MAPPING)) {
QueueMapping queueMapping =
new QueueMapping(mapping.getType(), mapping.getSource(),
user, groups.getGroups(user).get(0));
validateQueueMapping(queueMapping);
return getPlacementContext(queueMapping, user);
if (this.queueManager
.getQueue(groups.getGroups(user).get(0)) != null) {
QueueMapping queueMapping = new QueueMapping(mapping.getType(),
mapping.getSource(), user, groups.getGroups(user).get(0));
validateQueueMapping(queueMapping);
return getPlacementContext(queueMapping, user);
} else {
return null;
}
} else if (mapping.getParentQueue() != null
&& mapping.getParentQueue().equals(SECONDARY_GROUP_MAPPING)
&& mapping.getQueue().equals(CURRENT_USER_MAPPING)) {
@ -203,7 +207,13 @@ private ApplicationPlacementContext getPlacementForUser(String user)
} else if (mapping.queue.equals(CURRENT_USER_MAPPING)) {
return getPlacementContext(mapping, user);
} else if (mapping.queue.equals(PRIMARY_GROUP_MAPPING)) {
return getPlacementContext(mapping, groups.getGroups(user).get(0));
if (this.queueManager
.getQueue(groups.getGroups(user).get(0)) != null) {
return getPlacementContext(mapping,
groups.getGroups(user).get(0));
} else {
return null;
}
} else if (mapping.queue.equals(SECONDARY_GROUP_MAPPING)) {
String secondaryGroup = getSecondaryGroup(user);
if (secondaryGroup != null) {

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.placement;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@ -26,6 +27,7 @@
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.security.GroupMappingServiceProvider;
import org.apache.hadoop.security.Groups;
import org.apache.hadoop.security.NullGroupsMapping;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
@ -90,6 +92,7 @@ private void verifyQueueMapping(QueueMapping queueMapping, String inputUser,
when(queueManager.getQueue("agroup")).thenReturn(agroup);
when(queueManager.getQueue("bsubgroup2")).thenReturn(bsubgroup2);
when(queueManager.getQueue("asubgroup2")).thenReturn(asubgroup2);
rule.setQueueManager(queueManager);
ApplicationSubmissionContext asc = Records.newRecord(
ApplicationSubmissionContext.class);
@ -118,6 +121,20 @@ public void testSecondaryGroupMapping() throws YarnException {
"default");
}
@Test
public void testNullGroupMapping() throws YarnException {
conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
NullGroupsMapping.class, GroupMappingServiceProvider.class);
try {
verifyQueueMapping(
new QueueMapping(MappingType.USER, "%user", "%secondary_group"), "a",
"default");
fail("No Groups for user 'a'");
} catch (YarnException e) {
// Exception is expected as there are no groups for given user
}
}
@Test
public void testMapping() throws YarnException {
@ -131,6 +148,10 @@ public void testMapping() throws YarnException {
verifyQueueMapping(
new QueueMapping(MappingType.USER, "%user", "%primary_group"), "a",
"agroup");
// Queue "bgroup" is not configured, hence "default" should be used
verifyQueueMapping(
new QueueMapping(MappingType.USER, "%user", "%primary_group"), "b",
"default");
verifyQueueMapping(
new QueueMapping(MappingType.USER, "%user", "%user", "%primary_group"),
"a", YarnConfiguration.DEFAULT_QUEUE_NAME, "a", false, "agroup");

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
View File

@ -339,8 +339,8 @@ public static CapacitySchedulerConfiguration setupQueueConfiguration(
// Define top-level queues
// Set childQueue for root
conf.setQueues(ROOT,
new String[] {"a", "b", "c", "d", "esubgroup1", "asubgroup2",
"fgroup"});
new String[] {"a", "b", "c", "d", "esubgroup1", "esubgroup2", "fgroup",
"a1group", "ggroup", "g"});
conf.setCapacity(A, A_CAPACITY);
conf.setCapacity(B, B_CAPACITY);

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueMappingFactory.java
View File

@ -242,8 +242,8 @@ public void testNestedUserQueueWithPrimaryGroupAsDynamicParentQueue()
testNestedUserQueueWithDynamicParentQueue(queueMappingsForUG, true, "f");
try {
testNestedUserQueueWithDynamicParentQueue(queueMappingsForUG, true, "h");
fail("Leaf Queue 'h' doesn't exists");
testNestedUserQueueWithDynamicParentQueue(queueMappingsForUG, true, "g");
fail("Queue 'g' exists, but type is not Leaf Queue");
} catch (YarnException e) {
// Exception is expected as there is no such leaf queue
}