From 95e4ed18a6f3c79e0b306e7db886b5804d90bfbd Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Tue, 12 May 2020 09:03:50 +0200 Subject: [PATCH] YARN-10160. Add auto queue creation related configs to RMWebService#CapacitySchedulerQueueInfo. Contributed by Prabhu Joseph --- .../yarn/webapp/WebServicesTestUtils.java | 18 ++++ .../dao/CapacitySchedulerQueueInfo.java | 14 +++ .../webapp/dao/LeafQueueTemplateInfo.java | 91 +++++++++++++++++++ .../TestRMWebServicesCapacitySched.java | 36 +++++++- 4 files changed, 154 insertions(+), 5 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LeafQueueTemplateInfo.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java index 6d9e70fa32..b4832eff42 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java @@ -94,6 +94,24 @@ public static String getXmlString(Element element, String name) { return val; } + public static String getPropertyValue(Element element, String elementName, + String propertyName) { + NodeList id = element.getElementsByTagName(elementName); + Element line = (Element) id.item(0); + if (line == null) { + return null; + } + NodeList properties = line.getChildNodes(); + for (int i = 0; i < properties.getLength(); i++) { + Element property = (Element) properties.item(i); + if (getXmlString(property, "name").equals(propertyName)) { + return getXmlString(property, "value"); + } + } + return null; + } + + public static String getXmlAttrString(Element element, String name) { Attr at = element.getAttributeNode(name); if (at != null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index 947c44cc6b..e38ab0d0b4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -83,6 +83,8 @@ public class CapacitySchedulerQueueInfo { protected QueueAclsInfo queueAcls; protected int queuePriority; protected String orderingPolicyInfo; + protected boolean autoCreateChildQueueEnabled; + protected LeafQueueTemplateInfo leafQueueTemplate; CapacitySchedulerQueueInfo() { }; @@ -162,6 +164,10 @@ public class CapacitySchedulerQueueInfo { CapacitySchedulerConfiguration.getQueuePrefix(queuePath) + CAPACITY); isAbsoluteResource = (configuredCapacity != null) && RESOURCE_PATTERN.matcher(configuredCapacity).find(); + + autoCreateChildQueueEnabled = conf. + isAutoCreateChildQueueEnabled(queuePath); + leafQueueTemplate = new LeafQueueTemplateInfo(conf, queuePath); } protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) { @@ -291,4 +297,12 @@ public String getOrderingPolicyInfo() { public boolean isLeafQueue() { return getQueues() == null; } + + public boolean isAutoCreateChildQueueEnabled() { + return autoCreateChildQueueEnabled; + } + + public LeafQueueTemplateInfo getLeafQueueTemplate() { + return leafQueueTemplate; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LeafQueueTemplateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LeafQueueTemplateInfo.java new file mode 100644 index 0000000000..f528a47449 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LeafQueueTemplateInfo.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; + +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT; + +/** + * This class stores the LeafQueue Template configuration. + */ +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class LeafQueueTemplateInfo { + + private ArrayList property = new ArrayList<>(); + + public LeafQueueTemplateInfo() { + } // JAXB needs this + + public LeafQueueTemplateInfo(Configuration conf, String queuePath) { + String configPrefix = CapacitySchedulerConfiguration. + getQueuePrefix(queuePath) + AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX + + DOT; + conf.forEach(entry -> { + if (entry.getKey().startsWith(configPrefix)) { + String name = entry.getKey(); + int start = name.lastIndexOf(AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX + + DOT); + add(new ConfItem(name.substring(start), entry.getValue())); + } + }); + } + + public void add(ConfItem confItem) { + property.add(confItem); + } + + public ArrayList getItems() { + return property; + } + + /** + * This class stores the Configuration Property. + */ + @XmlAccessorType(XmlAccessType.FIELD) + public static class ConfItem { + + private String name; + private String value; + + public ConfItem() { + // JAXB needs this + } + + public ConfItem(String name, String value){ + this.name = name; + this.value = value; + } + + public String getKey() { + return name; + } + + public String getValue() { + return value; + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index 8098622a62..1e856bd412 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -24,6 +24,11 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.CAPACITY; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX; + import java.io.StringReader; import javax.ws.rs.core.MediaType; @@ -81,6 +86,7 @@ private class QueueInfo { String queueName; String state; boolean isAbsoluteResource; + boolean autoCreateChildQueueEnabled; } private class LeafQueueInfo extends QueueInfo { @@ -160,11 +166,17 @@ private static void setupQueueConfiguration( config.setCapacity(B3, 0.5f); config.setUserLimitFactor(B3, 100.0f); - config.setQueues(A1, new String[] {"a1a", "a1b"}); + config.setQueues(A1, new String[] {"a1a", "a1b", "a1c"}); final String A1A = A1 + ".a1a"; - config.setCapacity(A1A, 85); + config.setCapacity(A1A, 65); final String A1B = A1 + ".a1b"; config.setCapacity(A1B, 15); + final String A1C = A1 + ".a1c"; + config.setCapacity(A1C, 20); + + config.setAutoCreateChildQueueEnabled(A1C, true); + config.setInt(PREFIX + A1C + DOT + AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX + + DOT + CAPACITY, 50); } @Before @@ -295,6 +307,8 @@ public void verifySubQueueXML(Element qElem, String q, WebServicesTestUtils.getXmlInt(qElem, "numApplications"); qi.queueName = WebServicesTestUtils.getXmlString(qElem, "queueName"); qi.state = WebServicesTestUtils.getXmlString(qElem, "state"); + qi.autoCreateChildQueueEnabled = WebServicesTestUtils.getXmlBoolean(qElem, + "autoCreateChildQueueEnabled"); qi.isAbsoluteResource = WebServicesTestUtils.getXmlBoolean(qElem, "isAbsoluteResource"); verifySubQueueGeneric(q, qi, parentAbsCapacity, parentAbsMaxCapacity); @@ -311,6 +325,14 @@ public void verifySubQueueXML(Element qElem, String q, } } } + } else if (qi.autoCreateChildQueueEnabled) { + assertEquals("queueName doesn't match", "a1c", qi.queueName); + String capacityStr = WebServicesTestUtils.getPropertyValue(qElem, + "leafQueueTemplate", AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX + + DOT + CAPACITY); + int capacity = Integer.parseInt(capacityStr); + assertEquals(AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX + DOT + + CAPACITY + " doesn't match", 50, capacity); } else { LeafQueueInfo lqi = (LeafQueueInfo) qi; lqi.numActiveApplications = @@ -391,10 +413,10 @@ private void verifyClusterSchedulerGeneric(String type, float usedCapacity, private void verifySubQueue(JSONObject info, String q, float parentAbsCapacity, float parentAbsMaxCapacity) throws JSONException, Exception { - int numExpectedElements = 25; + int numExpectedElements = 27; boolean isParentQueue = true; if (!info.has("queues")) { - numExpectedElements = 43; + numExpectedElements = 45; isParentQueue = false; } assertEquals("incorrect number of elements", numExpectedElements, info.length()); @@ -412,7 +434,10 @@ private void verifySubQueue(JSONObject info, String q, verifySubQueueGeneric(q, qi, parentAbsCapacity, parentAbsMaxCapacity); - if (isParentQueue) { + // Separate Condition for Managed Parent Queue + if (qi.queueName.equals("a1c")) { + assertTrue(info.getBoolean("autoCreateChildQueueEnabled")); + } else if (isParentQueue) { JSONArray arr = info.getJSONObject("queues").getJSONArray("queue"); // test subqueues for (int i = 0; i < arr.length(); i++) { @@ -430,6 +455,7 @@ private void verifySubQueue(JSONObject info, String q, assertEquals("0", info.getString("queuePriority")); assertEquals("utilization", info.getString("orderingPolicyInfo")); + assertFalse(info.getBoolean("autoCreateChildQueueEnabled")); } else { Assert.assertEquals("\"type\" field is incorrect", "capacitySchedulerLeafQueueInfo", info.getString("type"));