big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10632. Make auto queue creation maximum allowed depth configurable (#3859)

Browse Source
This commit is contained in:
Andras Gyori 2022-01-11 07:38:59 +01:00 committed by GitHub
parent b88d5ae1c6
commit 21fe0e63fd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 63 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
View File

@ -426,6 +426,19 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
private ConfigurationProperties configurationProperties;
public int getMaximumAutoCreatedQueueDepth(String queuePath) {
return getInt(getQueuePrefix(queuePath) + MAXIMUM_QUEUE_DEPTH,
getInt(PREFIX + MAXIMUM_QUEUE_DEPTH, DEFAULT_MAXIMUM_QUEUE_DEPTH));
}
public void setMaximumAutoCreatedQueueDepth(String queuePath, int value) {
setInt(getQueuePrefix(queuePath) + MAXIMUM_QUEUE_DEPTH, value);
}
public void setMaximumAutoCreatedQueueDepth(int value) {
setInt(PREFIX + MAXIMUM_QUEUE_DEPTH, value);
}
/**
* Different resource types supported.
*/
@ -2136,6 +2149,13 @@ public void setDefaultLifetimePerQueue(String queue, long defaultLifetime) {
public static final int
DEFAULT_AUTO_QUEUE_CREATION_V2_MAX_QUEUES = 1000;
@Private
public static final String MAXIMUM_QUEUE_DEPTH =
AUTO_QUEUE_CREATION_V2_PREFIX + "maximum-queue-depth";
@Private
public static final int DEFAULT_MAXIMUM_QUEUE_DEPTH = 2;
@Private
public static final boolean DEFAULT_AUTO_QUEUE_CREATION_ENABLED = false;

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
View File

@ -23,6 +23,7 @@
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@ -72,7 +73,6 @@ public CSQueue hook(CSQueue queue) {
}
}
private static final int MAXIMUM_DYNAMIC_QUEUE_DEPTH = 2;
private static final QueueHook NOOP = new QueueHook();
private CapacitySchedulerContext csContext;
private final YarnAuthorizationProvider authorizer;
@ -592,14 +592,6 @@ public List<String> determineMissingParents(
&& parentCandidate.length() != 0) {
++firstStaticParentDistance;
if (firstStaticParentDistance > MAXIMUM_DYNAMIC_QUEUE_DEPTH) {
throw new SchedulerDynamicEditException(
"Could not auto create queue " + queue.getFullPath()
+ ". The distance of the LeafQueue from the first static " +
"ParentQueue is " + firstStaticParentDistance + ", which is " +
"above the limit.");
}
if (firstExistingParent == null) {
parentsToCreate.addFirst(parentCandidate.toString());
}
@ -614,6 +606,16 @@ public List<String> determineMissingParents(
firstExistingStaticParent = getQueue(parentCandidate.toString());
}
int maximumDepthOfStaticParent = csContext.getConfiguration().getMaximumAutoCreatedQueueDepth(
firstExistingStaticParent.getQueuePath());
if (firstStaticParentDistance > maximumDepthOfStaticParent) {
throw new SchedulerDynamicEditException(
"Could not auto create queue " + queue.getFullPath()
+ ". The distance of the LeafQueue from the first static " +
"ParentQueue is " + firstStaticParentDistance + ", which is " +
"above the limit.");
}
if (!(firstExistingParent instanceof ParentQueue)) {
throw new SchedulerDynamicEditException(
"Could not auto create hierarchy of "

35
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
View File

@ -318,11 +318,40 @@ public void testAutoCreateQueueWhenSiblingsNotInWeightMode()
createQueue("root.a.a2-auto");
}
@Test(expected = SchedulerDynamicEditException.class)
public void testAutoCreateQueueShouldFailIfDepthIsAboveLimit()
@Test()
public void testAutoCreateMaximumQueueDepth()
throws Exception {
startScheduler();
createQueue("root.a.a3-auto.a4-auto.a5-auto");
// By default, max depth is 2, therefore this is an invalid scenario
Assert.assertThrows(SchedulerDynamicEditException.class,
() -> createQueue("root.a.a3-auto.a4-auto.a5-auto"));
// Set depth 3 for root.a, making it a valid scenario
csConf.setMaximumAutoCreatedQueueDepth("root.a", 3);
cs.reinitialize(csConf, mockRM.getRMContext());
try {
createQueue("root.a.a3-auto.a4-auto.a5-auto");
} catch (SchedulerDynamicEditException sde) {
LOG.error("%s", sde);
Assert.fail("Depth is set for root.a, exception should not be thrown");
}
// Set global depth to 3
csConf.setMaximumAutoCreatedQueueDepth(3);
csConf.unset(CapacitySchedulerConfiguration.getQueuePrefix("root.a")
+ CapacitySchedulerConfiguration.MAXIMUM_QUEUE_DEPTH);
cs.reinitialize(csConf, mockRM.getRMContext());
try {
createQueue("root.a.a6-auto.a7-auto.a8-auto");
} catch (SchedulerDynamicEditException sde) {
LOG.error("%s", sde);
Assert.fail("Depth is set globally, exception should not be thrown");
}
// Set depth on a dynamic queue, which has no effect on auto queue creation validation
csConf.setMaximumAutoCreatedQueueDepth("root.a.a6-auto.a7-auto.a8-auto", 10);
Assert.assertThrows(SchedulerDynamicEditException.class,
() -> createQueue("root.a.a6-auto.a7-auto.a8-auto.a9-auto.a10-auto.a11-auto"));
}
@Test(expected = SchedulerDynamicEditException.class)