big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-11063. Support auto queue creation template wildcards for arbitrary queue depths. Contributed by Bence Kosztolnik.

Browse Source
This commit is contained in:
9uapaw 2022-07-28 17:32:10 +02:00
parent 95a85875d0
commit bf570bd4ac
2 changed files with 19 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java
View File

@ -42,7 +42,6 @@ public class AutoCreatedQueueTemplate {
AUTO_QUEUE_CREATION_V2_PREFIX + "parent-template."; AUTO_QUEUE_CREATION_V2_PREFIX + "parent-template.";
private static final String WILDCARD_QUEUE = "*"; private static final String WILDCARD_QUEUE = "*";
private static final int MAX_WILDCARD_LEVEL = 1;
private final Map<String, String> templateProperties = new HashMap<>(); private final Map<String, String> templateProperties = new HashMap<>();
private final Map<String, String> leafOnlyProperties = new HashMap<>(); private final Map<String, String> leafOnlyProperties = new HashMap<>();
@ -86,7 +85,7 @@ public Map<String, String> getParentOnlyProperties() {
/** /**
* Sets the common template properties and parent specific template * Sets the common template properties and parent specific template
* properties of a child queue based on its parent template settings. * properties of a child queue based on its parent template settings.
* @param conf configuration to set * @param conf configuration to set
* @param childQueuePath child queue path used for prefixing the properties * @param childQueuePath child queue path used for prefixing the properties
*/ */
public void setTemplateEntriesForChild(CapacitySchedulerConfiguration conf, public void setTemplateEntriesForChild(CapacitySchedulerConfiguration conf,
@ -169,8 +168,8 @@ private void setTemplateConfigEntries(CapacitySchedulerConfiguration configurati
// start with the most explicit format (without wildcard) // start with the most explicit format (without wildcard)
int wildcardLevel = 0; int wildcardLevel = 0;
// root can not be wildcarded // root can not be wildcarded
// MAX_WILDCARD_LEVEL will be configurable in the future int supportedWildcardLevel = Math.min(queuePathMaxIndex,
int supportedWildcardLevel = Math.min(queuePathMaxIndex, MAX_WILDCARD_LEVEL); configuration.getMaximumAutoCreatedQueueDepth(queuePath.getFullPath()));
// Allow root to have template properties // Allow root to have template properties
if (queuePath.isRoot()) { if (queuePath.isRoot()) {
supportedWildcardLevel = 0; supportedWildcardLevel = 0;

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java
View File

@ -66,6 +66,22 @@ public void testOneLevelWildcardTemplate() {
} }
@Test
public void testTwoLevelWildcardTemplate() {
conf.set(getTemplateKey("root.*", "capacity"), "6w");
conf.set(getTemplateKey("root.*.*", "capacity"), "5w");
new AutoCreatedQueueTemplate(conf, TEST_QUEUE_A)
.setTemplateEntriesForChild(conf, TEST_QUEUE_AB.getFullPath());
new AutoCreatedQueueTemplate(conf, TEST_QUEUE_AB)
.setTemplateEntriesForChild(conf, TEST_QUEUE_ABC.getFullPath());
Assert.assertEquals("weight is not set", 6f,
conf.getNonLabeledQueueWeight(TEST_QUEUE_AB.getFullPath()), 10e-6);
Assert.assertEquals("weight is not set", 5f,
conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC.getFullPath()), 10e-6);
}
@Test @Test
public void testIgnoredWhenRootWildcarded() { public void testIgnoredWhenRootWildcarded() {
conf.set(getTemplateKey("*", "capacity"), "6w"); conf.set(getTemplateKey("*", "capacity"), "6w");