big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10596. Allow static definition of childless ParentQueues with auto-queue-creation-v2 enabled. Contributed by Andras Gyori

Browse Source
This commit is contained in:
Szilard Nemeth 2021-01-26 16:22:35 +01:00
parent a9ff726e42
commit f1766e5bb4
3 changed files with 48 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
View File

@ -238,14 +238,21 @@ static CSQueue parseQueue(
boolean isReservableQueue = conf.isReservable(fullQueueName);
boolean isAutoCreateEnabled = conf.isAutoCreateChildQueueEnabled(
fullQueueName);
// if a queue is eligible for auto queue creation v2
// it must be a ParentQueue (even if it is empty)
boolean isAutoQueueCreationV2Enabled = conf.isAutoQueueCreationV2Enabled(
fullQueueName);
boolean isDynamicParent = false;
// Auto created parent queues might not have static children, but they
// must be kept as a ParentQueue
CSQueue oldQueue = oldQueues.get(fullQueueName);
if (oldQueue instanceof ParentQueue) {
isDynamicParent = ((ParentQueue) oldQueue).isDynamicQueue();
}
if (childQueueNames.size() == 0 && !isDynamicParent) {
if (childQueueNames.size() == 0 && !isDynamicParent &&
!isAutoQueueCreationV2Enabled) {
if (null == parent) {
throw new IllegalStateException(
"Queue configuration missing child queue names for " + queueName);

40
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
View File

@ -422,6 +422,46 @@ public void testAutoQueueCreationOnAppSubmission() throws Exception {
Assert.assertTrue(user0.isDynamicQueue());
}
@Test
public void testChildlessParentQueueWhenAutoQueueCreationEnabled()
throws Exception {
startScheduler();
csConf.setQueues("root", new String[]{"a", "b", "empty-auto-parent"});
csConf.setNonLabeledQueueWeight("root", 1f);
csConf.setNonLabeledQueueWeight("root.a", 1f);
csConf.setNonLabeledQueueWeight("root.b", 1f);
csConf.setQueues("root.a", new String[]{"a1"});
csConf.setNonLabeledQueueWeight("root.a.a1", 1f);
csConf.setAutoQueueCreationV2Enabled("root", true);
csConf.setAutoQueueCreationV2Enabled("root.a", true);
cs.reinitialize(csConf, mockRM.getRMContext());
CSQueue empty = cs.getQueue("root.empty-auto-parent");
Assert.assertTrue("empty-auto-parent is not a LeafQueue",
empty instanceof LeafQueue);
empty.stopQueue();
csConf.setQueues("root", new String[]{"a", "b", "empty-auto-parent"});
csConf.setNonLabeledQueueWeight("root", 1f);
csConf.setNonLabeledQueueWeight("root.a", 1f);
csConf.setNonLabeledQueueWeight("root.b", 1f);
csConf.setQueues("root.a", new String[]{"a1"});
csConf.setNonLabeledQueueWeight("root.a.a1", 1f);
csConf.setAutoQueueCreationV2Enabled("root", true);
csConf.setAutoQueueCreationV2Enabled("root.a", true);
csConf.setAutoQueueCreationV2Enabled("root.empty-auto-parent", true);
cs.reinitialize(csConf, mockRM.getRMContext());
empty = cs.getQueue("root.empty-auto-parent");
Assert.assertTrue("empty-auto-parent is not a ParentQueue",
empty instanceof ParentQueue);
Assert.assertEquals("empty-auto-parent has children",
0, empty.getChildQueues().size());
Assert.assertTrue("empty-auto-parent is not eligible " +
"for auto queue creation",
((ParentQueue)empty).isEligibleForAutoQueueCreation());
}
private LeafQueue createQueue(String queuePath) throws YarnException {
return autoQueueHandler.autoCreateQueue(
CSQueueUtils.extractQueuePath(queuePath));

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java
View File

@ -507,8 +507,6 @@ private static Configuration createWeightConfigInternal(boolean enableAqc) {
if (enableAqc) {
conf.put("yarn.scheduler.capacity.root.auto-queue-creation-v2.enabled",
"true");
conf.put("yarn.scheduler.capacity.root.default." +
"auto-queue-creation-v2.enabled", "true");
}
return createConfiguration(conf);
}