big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10160. Add auto queue creation related configs to RMWebService#CapacitySchedulerQueueInfo. Contributed by Prabhu Joseph

Browse Source
This commit is contained in:
Szilard Nemeth 2020-05-12 09:03:50 +02:00
parent d1e5e393c3
commit 95e4ed18a6
4 changed files with 154 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java
View File

@ -94,6 +94,24 @@ public static String getXmlString(Element element, String name) {
return val;
}
public static String getPropertyValue(Element element, String elementName,
String propertyName) {
NodeList id = element.getElementsByTagName(elementName);
Element line = (Element) id.item(0);
if (line == null) {
return null;
}
NodeList properties = line.getChildNodes();
for (int i = 0; i < properties.getLength(); i++) {
Element property = (Element) properties.item(i);
if (getXmlString(property, "name").equals(propertyName)) {
return getXmlString(property, "value");
}
}
return null;
}
public static String getXmlAttrString(Element element, String name) {
Attr at = element.getAttributeNode(name);
if (at != null) {

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
View File

@ -83,6 +83,8 @@ public class CapacitySchedulerQueueInfo {
protected QueueAclsInfo queueAcls;
protected int queuePriority;
protected String orderingPolicyInfo;
protected boolean autoCreateChildQueueEnabled;
protected LeafQueueTemplateInfo leafQueueTemplate;
CapacitySchedulerQueueInfo() {
};
@ -162,6 +164,10 @@ public class CapacitySchedulerQueueInfo {
CapacitySchedulerConfiguration.getQueuePrefix(queuePath) + CAPACITY);
isAbsoluteResource = (configuredCapacity != null)
&& RESOURCE_PATTERN.matcher(configuredCapacity).find();
autoCreateChildQueueEnabled = conf.
isAutoCreateChildQueueEnabled(queuePath);
leafQueueTemplate = new LeafQueueTemplateInfo(conf, queuePath);
}
protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) {
@ -291,4 +297,12 @@ public String getOrderingPolicyInfo() {
public boolean isLeafQueue() {
return getQueues() == null;
}
public boolean isAutoCreateChildQueueEnabled() {
return autoCreateChildQueueEnabled;
}
public LeafQueueTemplateInfo getLeafQueueTemplate() {
return leafQueueTemplate;
}
}

91
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LeafQueueTemplateInfo.java Normal file
View File

@ -0,0 +1,91 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import java.util.ArrayList;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT;
/**
* This class stores the LeafQueue Template configuration.
*/
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
public class LeafQueueTemplateInfo {
private ArrayList<ConfItem> property = new ArrayList<>();
public LeafQueueTemplateInfo() {
} // JAXB needs this
public LeafQueueTemplateInfo(Configuration conf, String queuePath) {
String configPrefix = CapacitySchedulerConfiguration.
getQueuePrefix(queuePath) + AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX
+ DOT;
conf.forEach(entry -> {
if (entry.getKey().startsWith(configPrefix)) {
String name = entry.getKey();
int start = name.lastIndexOf(AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX
+ DOT);
add(new ConfItem(name.substring(start), entry.getValue()));
}
});
}
public void add(ConfItem confItem) {
property.add(confItem);
}
public ArrayList<ConfItem> getItems() {
return property;
}
/**
* This class stores the Configuration Property.
*/
@XmlAccessorType(XmlAccessType.FIELD)
public static class ConfItem {
private String name;
private String value;
public ConfItem() {
// JAXB needs this
}
public ConfItem(String name, String value){
this.name = name;
this.value = value;
}
public String getKey() {
return name;
}
public String getValue() {
return value;
}
}
}

36
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
View File

@ -24,6 +24,11 @@
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.CAPACITY;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX;
import java.io.StringReader;
import javax.ws.rs.core.MediaType;
@ -81,6 +86,7 @@ private class QueueInfo {
String queueName;
String state;
boolean isAbsoluteResource;
boolean autoCreateChildQueueEnabled;
}
private class LeafQueueInfo extends QueueInfo {
@ -160,11 +166,17 @@ private static void setupQueueConfiguration(
config.setCapacity(B3, 0.5f);
config.setUserLimitFactor(B3, 100.0f);
config.setQueues(A1, new String[] {"a1a", "a1b"});
config.setQueues(A1, new String[] {"a1a", "a1b", "a1c"});
final String A1A = A1 + ".a1a";
config.setCapacity(A1A, 85);
config.setCapacity(A1A, 65);
final String A1B = A1 + ".a1b";
config.setCapacity(A1B, 15);
final String A1C = A1 + ".a1c";
config.setCapacity(A1C, 20);
config.setAutoCreateChildQueueEnabled(A1C, true);
config.setInt(PREFIX + A1C + DOT + AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX
+ DOT + CAPACITY, 50);
}
@Before
@ -295,6 +307,8 @@ public void verifySubQueueXML(Element qElem, String q,
WebServicesTestUtils.getXmlInt(qElem, "numApplications");
qi.queueName = WebServicesTestUtils.getXmlString(qElem, "queueName");
qi.state = WebServicesTestUtils.getXmlString(qElem, "state");
qi.autoCreateChildQueueEnabled = WebServicesTestUtils.getXmlBoolean(qElem,
"autoCreateChildQueueEnabled");
qi.isAbsoluteResource = WebServicesTestUtils.getXmlBoolean(qElem,
"isAbsoluteResource");
verifySubQueueGeneric(q, qi, parentAbsCapacity, parentAbsMaxCapacity);
@ -311,6 +325,14 @@ public void verifySubQueueXML(Element qElem, String q,
}
}
}
} else if (qi.autoCreateChildQueueEnabled) {
assertEquals("queueName doesn't match", "a1c", qi.queueName);
String capacityStr = WebServicesTestUtils.getPropertyValue(qElem,
"leafQueueTemplate", AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX
+ DOT + CAPACITY);
int capacity = Integer.parseInt(capacityStr);
assertEquals(AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX + DOT
+ CAPACITY + " doesn't match", 50, capacity);
} else {
LeafQueueInfo lqi = (LeafQueueInfo) qi;
lqi.numActiveApplications =
@ -391,10 +413,10 @@ private void verifyClusterSchedulerGeneric(String type, float usedCapacity,
private void verifySubQueue(JSONObject info, String q,
float parentAbsCapacity, float parentAbsMaxCapacity)
throws JSONException, Exception {
int numExpectedElements = 25;
int numExpectedElements = 27;
boolean isParentQueue = true;
if (!info.has("queues")) {
numExpectedElements = 43;
numExpectedElements = 45;
isParentQueue = false;
}
assertEquals("incorrect number of elements", numExpectedElements, info.length());
@ -412,7 +434,10 @@ private void verifySubQueue(JSONObject info, String q,
verifySubQueueGeneric(q, qi, parentAbsCapacity, parentAbsMaxCapacity);
if (isParentQueue) {
// Separate Condition for Managed Parent Queue
if (qi.queueName.equals("a1c")) {
assertTrue(info.getBoolean("autoCreateChildQueueEnabled"));
} else if (isParentQueue) {
JSONArray arr = info.getJSONObject("queues").getJSONArray("queue");
// test subqueues
for (int i = 0; i < arr.length(); i++) {
@ -430,6 +455,7 @@ private void verifySubQueue(JSONObject info, String q,
assertEquals("0", info.getString("queuePriority"));
assertEquals("utilization", info.getString("orderingPolicyInfo"));
assertFalse(info.getBoolean("autoCreateChildQueueEnabled"));
} else {
Assert.assertEquals("\"type\" field is incorrect",
"capacitySchedulerLeafQueueInfo", info.getString("type"));