big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10783. Allow definition of auto queue template properties in root. Contributed by Andras Gyori

Browse Source
This commit is contained in:
Szilard Nemeth 2021-05-25 13:55:59 +02:00
parent 1cbcde04f3
commit 2541efa496
2 changed files with 22 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java
View File

@ -29,6 +29,7 @@
import java.util.Set;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AUTO_QUEUE_CREATION_V2_PREFIX;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT;
/**
* A handler for storing and setting auto created queue template settings.
@ -68,6 +69,10 @@ public Map<String, String> getTemplateProperties() {
*/
public void setTemplateEntriesForChild(Configuration conf,
String childQueuePath) {
if (childQueuePath.equals(ROOT)) {
return;
}
// Get all properties that are explicitly set
Set<String> alreadySetProps = conf.getPropsWithPrefix(
CapacitySchedulerConfiguration.getQueuePrefix(childQueuePath)).keySet();
@ -94,8 +99,8 @@ private void setTemplateConfigEntries(Configuration configuration,
List<String> queuePathParts = new ArrayList<>(Arrays.asList(
queuePath.split("\\.")));
if (queuePathParts.size() <= 1) {
// This is either root or an empty queue name
if (queuePathParts.size() <= 1 && !queuePath.equals(ROOT)) {
// This is an invalid queue path
return;
}
int queuePathMaxIndex = queuePathParts.size() - 1;
@ -106,7 +111,10 @@ private void setTemplateConfigEntries(Configuration configuration,
// MAX_WILDCARD_LEVEL will be configurable in the future
int supportedWildcardLevel = Math.min(queuePathMaxIndex - 1,
MAX_WILDCARD_LEVEL);
// Allow root to have template properties
if (queuePath.equals(ROOT)) {
supportedWildcardLevel = 0;
}
// Collect all template entries
while (wildcardLevel <= supportedWildcardLevel) {

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java
View File

@ -109,6 +109,17 @@ public void testTemplatePrecedence() {
newConf.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6);
}
@Test
public void testRootTemplate() {
conf.set(getTemplateKey("root", "capacity"), "2w");
AutoCreatedQueueTemplate template =
new AutoCreatedQueueTemplate(conf, ROOT);
template.setTemplateEntriesForChild(conf, TEST_QUEUE_A);
Assert.assertEquals("root property is not set", 2f,
conf.getNonLabeledQueueWeight(TEST_QUEUE_A), 10e-6);
}
private String getTemplateKey(String queuePath, String entryKey) {
return CapacitySchedulerConfiguration.getQueuePrefix(queuePath)
+ AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX + entryKey;