big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10409. Improve MockQueueHierarchyBuilder to detect queue ambiguity. Contributed by Gergely Pollak

Browse Source
This commit is contained in:
Szilard Nemeth 2020-08-29 15:24:50 +02:00
parent 44542863f4
commit a888d580d8
2 changed files with 20 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/MockQueueHierarchyBuilder.java
View File

@ -19,11 +19,13 @@
package org.apache.hadoop.yarn.server.resourcemanager.placement;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.commons.compress.utils.Lists;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.*;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@ -33,6 +35,8 @@ class MockQueueHierarchyBuilder {
private static final String QUEUE_SEP = ".";
private List<String> queuePaths = Lists.newArrayList();
private List<String> managedParentQueues = Lists.newArrayList();
private Set<String> ambiguous = Sets.newHashSet();
private Map<String, String> shortNameMapping = Maps.newHashMap();
private CapacitySchedulerQueueManager queueManager;
public static MockQueueHierarchyBuilder create() {
@ -75,6 +79,14 @@ public void build() {
for (String queuePath : queuePaths) {
addQueues(queues, queuePath);
}
ambiguous.forEach(queue -> {
if (queue.equals("root")) {
return;
}
when(queueManager.isAmbiguous(queue)).thenReturn(true);
when(queueManager.getQueue(queue)).thenReturn(null);
});
}
private void addQueues(Map<String, AbstractCSQueue> queues,
@ -89,6 +101,12 @@ private void addQueues(Map<String, AbstractCSQueue> queues,
currentQueuePath += currentQueuePath.equals("") ?
queueName : QUEUE_SEP + queueName;
if (shortNameMapping.containsKey(queueName) &&
!shortNameMapping.get(queueName).equals(currentQueuePath)) {
ambiguous.add(queueName);
}
shortNameMapping.put(queueName, currentQueuePath);
if (managedParentQueues.contains(parentPath) && !isLeaf) {
throw new IllegalStateException("Cannot add a queue under " +
"managed parent");

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java
View File

@ -84,7 +84,6 @@ private void verifyQueueMapping(QueueMappingTestData queueMappingTestData)
.build();
when(queueManager.getQueue(isNull())).thenReturn(null);
when(queueManager.isAmbiguous("primarygrouponly")).thenReturn(true);
rule.setQueueManager(queueManager);
ApplicationSubmissionContext asc = Records.newRecord(
ApplicationSubmissionContext.class);
@ -349,7 +348,7 @@ public void testGenericUserMappingWithNoSecondaryGroup()
.build());
}
@Test(expected = YarnException.class)
@Test
public void testUserMappingToNestedUserPrimaryGroupWithAmbiguousQueues()
throws YarnException {
// u:%user:%user, submitter nosecondarygroupuser, queue is ambiguous
@ -362,6 +361,7 @@ public void testUserMappingToNestedUserPrimaryGroupWithAmbiguousQueues()
.parentQueue("%primary_group")
.build())
.inputUser("nosecondarygroupuser")
.expectedQueue("default")
.build());
}