From 1b9efe58c90a4485308387fe34d8b50756b717a3 Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Wed, 28 Jul 2021 16:50:14 +0200 Subject: [PATCH] YARN-10790. CS Flexible AQC: Add separate parent and leaf template property. Contributed by Andras Gyori --- .../capacity/AutoCreatedQueueTemplate.java | 104 ++++++++++++++++-- .../webapp/dao/CapacitySchedulerInfo.java | 14 ++- .../dao/CapacitySchedulerQueueInfo.java | 16 ++- .../helper/CapacitySchedulerInfoHelper.java | 5 +- .../TestAutoCreatedQueueTemplate.java | 41 +++++++ .../TestRMWebServicesCapacitySched.java | 6 +- .../TestRMWebServicesForCSWithPartitions.java | 2 +- 7 files changed, 165 insertions(+), 23 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java index 203ec4dcd6..cb0f7891f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java @@ -30,6 +30,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AUTO_QUEUE_CREATION_V2_PREFIX; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.getQueuePrefix; /** * A handler for storing and setting auto created queue template settings. @@ -37,10 +38,17 @@ public class AutoCreatedQueueTemplate { public static final String AUTO_QUEUE_TEMPLATE_PREFIX = AUTO_QUEUE_CREATION_V2_PREFIX + "template."; + public static final String AUTO_QUEUE_LEAF_TEMPLATE_PREFIX = + AUTO_QUEUE_CREATION_V2_PREFIX + "leaf-template."; + public static final String AUTO_QUEUE_PARENT_TEMPLATE_PREFIX = + AUTO_QUEUE_CREATION_V2_PREFIX + "parent-template."; + private static final String WILDCARD_QUEUE = "*"; private static final int MAX_WILDCARD_LEVEL = 1; private final Map templateProperties = new HashMap<>(); + private final Map leafOnlyProperties = new HashMap<>(); + private final Map parentOnlyProperties = new HashMap<>(); public AutoCreatedQueueTemplate(Configuration configuration, String queuePath) { @@ -54,7 +62,7 @@ public static String getAutoQueueTemplatePrefix(String queue) { } /** - * Get the template properties attached to a parent queue. + * Get the common template properties specified for a parent queue. * @return template property names and values */ public Map getTemplateProperties() { @@ -62,13 +70,44 @@ public Map getTemplateProperties() { } /** - * Sets the configuration properties of a child queue based on its parent - * template settings. - * @param conf configuration to set + * Get the leaf specific template properties specified for a parent queue. + * @return template property names and values + */ + public Map getLeafOnlyProperties() { + return leafOnlyProperties; + } + + /** + * Get the parent specific template properties specified for a parent queue. + * @return template property names and values + */ + public Map getParentOnlyProperties() { + return parentOnlyProperties; + } + + /** + * Sets the common template properties and parent specific template + * properties of a child queue based on its parent template settings. + * @param conf configuration to set * @param childQueuePath child queue path used for prefixing the properties */ public void setTemplateEntriesForChild(Configuration conf, String childQueuePath) { + setTemplateEntriesForChild(conf, childQueuePath, false); + } + + /** + * Sets the common template properties and leaf or parent specific template + * properties of a child queue based on its parent template settings. + * template settings. + * @param conf configuration to set + * @param isLeaf whether to include leaf specific template properties, or + * parent specific template properties + * @param childQueuePath child queue path used for prefixing the properties + */ + public void setTemplateEntriesForChild(Configuration conf, + String childQueuePath, + boolean isLeaf) { if (childQueuePath.equals(ROOT)) { return; } @@ -77,7 +116,14 @@ public void setTemplateEntriesForChild(Configuration conf, Set alreadySetProps = conf.getPropsWithPrefix( CapacitySchedulerConfiguration.getQueuePrefix(childQueuePath)).keySet(); - for (Map.Entry entry : templateProperties.entrySet()) { + // Check template properties only set for leaf or parent queues + Map queueTypeSpecificTemplates = parentOnlyProperties; + if (isLeaf) { + queueTypeSpecificTemplates = leafOnlyProperties; + } + + for (Map.Entry entry : + queueTypeSpecificTemplates.entrySet()) { // Do not overwrite explicitly configured properties if (alreadySetProps.contains(entry.getKey())) { continue; @@ -85,6 +131,17 @@ public void setTemplateEntriesForChild(Configuration conf, conf.set(CapacitySchedulerConfiguration.getQueuePrefix( childQueuePath) + entry.getKey(), entry.getValue()); } + + for (Map.Entry entry : templateProperties.entrySet()) { + // Do not overwrite explicitly configured properties or properties set + // by queue type specific templates (parent-template and leaf-template) + if (alreadySetProps.contains(entry.getKey()) + || queueTypeSpecificTemplates.containsKey(entry.getKey())) { + continue; + } + conf.set(CapacitySchedulerConfiguration.getQueuePrefix( + childQueuePath) + entry.getKey(), entry.getValue()); + } } /** @@ -118,16 +175,15 @@ private void setTemplateConfigEntries(Configuration configuration, // Collect all template entries while (wildcardLevel <= supportedWildcardLevel) { - // Get all config entries with the specified prefix String templateQueuePath = String.join(".", queuePathParts); // Get all configuration entries with - // .auto-queue-creation-v2.template prefix - Map props = configuration.getPropsWithPrefix( - getAutoQueueTemplatePrefix(templateQueuePath)); + // yarn.scheduler.capacity. prefix + Map queueProps = configuration.getPropsWithPrefix( + getQueuePrefix(templateQueuePath)); - for (Map.Entry entry : props.entrySet()) { - // If an entry is already present, it had a higher precedence - templateProperties.putIfAbsent(entry.getKey(), entry.getValue()); + // Store template, parent-template and leaf-template properties + for (Map.Entry entry : queueProps.entrySet()) { + storeConfiguredTemplates(entry.getKey(), entry.getValue()); } // Replace a queue part with a wildcard based on the wildcard level @@ -138,4 +194,28 @@ private void setTemplateConfigEntries(Configuration configuration, ++wildcardLevel; } } + + private void storeConfiguredTemplates( + String templateKey, String templateValue) { + String prefix = ""; + Map properties = templateProperties; + + if (templateKey.startsWith(AUTO_QUEUE_TEMPLATE_PREFIX)) { + prefix = AUTO_QUEUE_TEMPLATE_PREFIX; + } else if (templateKey.startsWith(AUTO_QUEUE_LEAF_TEMPLATE_PREFIX)) { + prefix = AUTO_QUEUE_LEAF_TEMPLATE_PREFIX; + properties = leafOnlyProperties; + } else if (templateKey.startsWith( + AUTO_QUEUE_PARENT_TEMPLATE_PREFIX)) { + prefix = AUTO_QUEUE_PARENT_TEMPLATE_PREFIX; + properties = parentOnlyProperties; + } + + if (!prefix.isEmpty()) { + // Trim template prefix from key + String key = templateKey.substring(prefix.length()); + // If an entry is already present, it had a higher precedence + properties.putIfAbsent(key, templateValue); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java index cc02c23fd7..79d11e2b96 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java @@ -63,6 +63,8 @@ public class CapacitySchedulerInfo extends SchedulerInfo { protected String autoCreationEligibility; protected String defaultNodeLabelExpression; protected AutoQueueTemplatePropertiesInfo autoQueueTemplateProperties; + protected AutoQueueTemplatePropertiesInfo autoQueueParentTemplateProperties; + protected AutoQueueTemplatePropertiesInfo autoQueueLeafTemplateProperties; @XmlTransient static final float EPSILON = 1e-8f; @@ -108,10 +110,18 @@ public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs) { queuePriority = parent.getPriority().getPriority(); if (parent instanceof ParentQueue) { - orderingPolicyInfo = ((ParentQueue) parent).getQueueOrderingPolicy() + ParentQueue queue = (ParentQueue) parent; + orderingPolicyInfo = queue.getQueueOrderingPolicy() .getConfigName(); autoQueueTemplateProperties = CapacitySchedulerInfoHelper - .getAutoCreatedTemplate((ParentQueue) parent); + .getAutoCreatedTemplate(queue.getAutoCreatedQueueTemplate() + .getTemplateProperties()); + autoQueueParentTemplateProperties = CapacitySchedulerInfoHelper + .getAutoCreatedTemplate(queue.getAutoCreatedQueueTemplate() + .getParentOnlyProperties()); + autoQueueLeafTemplateProperties = CapacitySchedulerInfoHelper + .getAutoCreatedTemplate(queue.getAutoCreatedQueueTemplate() + .getLeafOnlyProperties()); } mode = CapacitySchedulerInfoHelper.getMode(parent); queueType = CapacitySchedulerInfoHelper.getQueueType(parent); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index 152620d660..452fa50343 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -92,6 +92,10 @@ public class CapacitySchedulerQueueInfo { protected String defaultNodeLabelExpression; protected AutoQueueTemplatePropertiesInfo autoQueueTemplateProperties = new AutoQueueTemplatePropertiesInfo(); + protected AutoQueueTemplatePropertiesInfo autoQueueParentTemplateProperties = + new AutoQueueTemplatePropertiesInfo(); + protected AutoQueueTemplatePropertiesInfo autoQueueLeafTemplateProperties = + new AutoQueueTemplatePropertiesInfo(); CapacitySchedulerQueueInfo() { }; @@ -172,10 +176,18 @@ public class CapacitySchedulerQueueInfo { queuePriority = q.getPriority().getPriority(); if (q instanceof ParentQueue) { - orderingPolicyInfo = ((ParentQueue) q).getQueueOrderingPolicy() + ParentQueue queue = (ParentQueue) q; + orderingPolicyInfo = queue.getQueueOrderingPolicy() .getConfigName(); autoQueueTemplateProperties = CapacitySchedulerInfoHelper - .getAutoCreatedTemplate((ParentQueue) q); + .getAutoCreatedTemplate(queue.getAutoCreatedQueueTemplate() + .getTemplateProperties()); + autoQueueParentTemplateProperties = CapacitySchedulerInfoHelper + .getAutoCreatedTemplate(queue.getAutoCreatedQueueTemplate() + .getParentOnlyProperties()); + autoQueueLeafTemplateProperties = CapacitySchedulerInfoHelper + .getAutoCreatedTemplate(queue.getAutoCreatedQueueTemplate() + .getLeafOnlyProperties()); } String configuredCapacity = conf.get( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java index f71928b61b..8b3602da31 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/helper/CapacitySchedulerInfoHelper.java @@ -112,12 +112,11 @@ public static String getAutoCreationEligibility(CSQueue queue) { } public static AutoQueueTemplatePropertiesInfo getAutoCreatedTemplate( - ParentQueue parent) { + Map templateProperties) { AutoQueueTemplatePropertiesInfo propertiesInfo = new AutoQueueTemplatePropertiesInfo(); for (Map.Entry e : - parent.getAutoCreatedQueueTemplate().getTemplateProperties() - .entrySet()) { + templateProperties.entrySet()) { propertiesInfo.add(new ConfItem(e.getKey(), e.getValue())); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java index 2763af026a..37f1378d7c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java @@ -22,10 +22,13 @@ import org.junit.Before; import org.junit.Test; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AUTO_CREATE_CHILD_QUEUE_AUTO_REMOVAL_ENABLE; + public class TestAutoCreatedQueueTemplate { private static final String TEST_QUEUE_ABC = "root.a.b.c"; private static final String TEST_QUEUE_AB = "root.a.b"; private static final String TEST_QUEUE_A = "root.a"; + private static final String TEST_QUEUE_B = "root.b"; private static final String ROOT = "root"; private CapacitySchedulerConfiguration conf; @@ -120,8 +123,46 @@ public void testRootTemplate() { conf.getNonLabeledQueueWeight(TEST_QUEUE_A), 10e-6); } + @Test + public void testQueueSpecificTemplates() { + conf.set(getTemplateKey("root", "capacity"), "2w"); + conf.set(getLeafTemplateKey("root", + "default-node-label-expression"), "test"); + conf.set(getLeafTemplateKey("root", "capacity"), "10w"); + conf.setBoolean(getParentTemplateKey( + "root", AUTO_CREATE_CHILD_QUEUE_AUTO_REMOVAL_ENABLE), false); + + AutoCreatedQueueTemplate template = + new AutoCreatedQueueTemplate(conf, ROOT); + template.setTemplateEntriesForChild(conf, TEST_QUEUE_A); + template.setTemplateEntriesForChild(conf, TEST_QUEUE_B, true); + + Assert.assertNull("default-node-label-expression is set for parent", + conf.getDefaultNodeLabelExpression(TEST_QUEUE_A)); + Assert.assertEquals("default-node-label-expression is not set for leaf", + "test", conf.getDefaultNodeLabelExpression(TEST_QUEUE_B)); + Assert.assertFalse("auto queue removal is not disabled for parent", + conf.isAutoExpiredDeletionEnabled(TEST_QUEUE_A)); + Assert.assertEquals("weight should not be overridden when set by " + + "queue type specific template", + 10f, conf.getNonLabeledQueueWeight(TEST_QUEUE_B), 10e-6); + Assert.assertEquals("weight should be set by common template", + 2f, conf.getNonLabeledQueueWeight(TEST_QUEUE_A), 10e-6); + + } + private String getTemplateKey(String queuePath, String entryKey) { return CapacitySchedulerConfiguration.getQueuePrefix(queuePath) + AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX + entryKey; } + + private String getParentTemplateKey(String queuePath, String entryKey) { + return CapacitySchedulerConfiguration.getQueuePrefix(queuePath) + + AutoCreatedQueueTemplate.AUTO_QUEUE_PARENT_TEMPLATE_PREFIX + entryKey; + } + + private String getLeafTemplateKey(String queuePath, String entryKey) { + return CapacitySchedulerConfiguration.getQueuePrefix(queuePath) + + AutoCreatedQueueTemplate.AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + entryKey; + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index 459645facc..b472e099f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -369,7 +369,7 @@ private void verifyClusterScheduler(JSONObject json) throws JSONException, JSONObject info = json.getJSONObject("scheduler"); assertEquals("incorrect number of elements in: " + info, 1, info.length()); info = info.getJSONObject("schedulerInfo"); - assertEquals("incorrect number of elements in: " + info, 20, info.length()); + assertEquals("incorrect number of elements in: " + info, 22, info.length()); verifyClusterSchedulerGeneric(info.getString("type"), (float) info.getDouble("usedCapacity"), (float) info.getDouble("capacity"), @@ -424,10 +424,10 @@ private void verifyClusterSchedulerGeneric(String type, float usedCapacity, private void verifySubQueue(JSONObject info, String q, float parentAbsCapacity, float parentAbsMaxCapacity) throws JSONException, Exception { - int numExpectedElements = 35; + int numExpectedElements = 37; boolean isParentQueue = true; if (!info.has("queues")) { - numExpectedElements = 53; + numExpectedElements = 55; isParentQueue = false; } assertEquals("incorrect number of elements", numExpectedElements, info.length()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java index 2e67d69894..36b27a2323 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java @@ -574,7 +574,7 @@ private void verifySchedulerInfoJson(JSONObject json) JSONObject info = json.getJSONObject("scheduler"); assertEquals("incorrect number of elements", 1, info.length()); info = info.getJSONObject("schedulerInfo"); - assertEquals("incorrect number of elements", 20, info.length()); + assertEquals("incorrect number of elements", 22, info.length()); JSONObject capacitiesJsonObject = info.getJSONObject(CAPACITIES); JSONArray partitionsCapsArray = capacitiesJsonObject.getJSONArray(QUEUE_CAPACITIES_BY_PARTITION);