big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-11023. Extend the root QueueInfo with max-parallel-apps in CapacityScheduler. Contributed by Tamas Domok

Browse Source
This commit is contained in:
Szilard Nemeth 2021-12-07 15:26:07 +01:00
parent 79cc7b46af
commit 60440a836a
3 changed files with 10 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
View File

@ -50,6 +50,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
protected float normalizedWeight;
protected String queueName;
private String queuePath;
protected int maxParallelApps;
protected CapacitySchedulerQueueInfoList queues;
protected QueueCapacitiesInfo capacities;
protected CapacitySchedulerHealthInfo health;
@ -83,6 +84,7 @@ public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs) {
this.maxCapacity = max * 100;
this.weight = parent.getQueueCapacities().getWeight();
this.normalizedWeight = parent.getQueueCapacities().getNormalizedWeight();
this.maxParallelApps = parent.getMaxParallelApps();
capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(),
parent.getQueueResourceQuotas(), false);

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
View File

@ -270,7 +270,8 @@ public void verifyClusterSchedulerXML(NodeList nodes) throws Exception {
WebServicesTestUtils.getXmlFloat(element, "capacity"),
WebServicesTestUtils.getXmlFloat(element, "maxCapacity"),
WebServicesTestUtils.getXmlString(element, "queueName"),
WebServicesTestUtils.getXmlString(element, "queuePath"));
WebServicesTestUtils.getXmlString(element, "queuePath"),
WebServicesTestUtils.getXmlInt(element, "maxParallelApps"));
NodeList children = element.getChildNodes();
for (int j = 0; j < children.getLength(); j++) {
@ -374,13 +375,14 @@ private void verifyClusterScheduler(JSONObject json) throws JSONException,
JSONObject info = json.getJSONObject("scheduler");
assertEquals("incorrect number of elements in: " + info, 1, info.length());
info = info.getJSONObject("schedulerInfo");
assertEquals("incorrect number of elements in: " + info, 22, info.length());
assertEquals("incorrect number of elements in: " + info, 23, info.length());
verifyClusterSchedulerGeneric(info.getString("type"),
(float) info.getDouble("usedCapacity"),
(float) info.getDouble("capacity"),
(float) info.getDouble("maxCapacity"),
info.getString("queueName"),
info.getString("queuePath"));
info.getString("queuePath"),
info.getInt("maxParallelApps"));
JSONObject health = info.getJSONObject("health");
assertNotNull(health);
assertEquals("incorrect number of elements in: " + health, 3,
@ -415,7 +417,7 @@ private void verifyClusterScheduler(JSONObject json) throws JSONException,
}
private void verifyClusterSchedulerGeneric(String type, float usedCapacity,
float capacity, float maxCapacity, String queueName, String queuePath)
float capacity, float maxCapacity, String queueName, String queuePath, int maxParallelApps)
throws Exception {
assertTrue("type doesn't match", "capacityScheduler".matches(type));
@ -424,6 +426,7 @@ private void verifyClusterSchedulerGeneric(String type, float usedCapacity,
assertEquals("maxCapacity doesn't match", 100, maxCapacity, 1e-3f);
assertTrue("queueName doesn't match", "root".matches(queueName));
assertTrue("queuePath doesn't match", "root".matches(queuePath));
assertEquals("maxParallelApps doesn't match ", Integer.MAX_VALUE, maxParallelApps);
}
private void verifySubQueue(JSONObject info, String q,

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java
View File

@ -574,7 +574,7 @@ private void verifySchedulerInfoJson(JSONObject json)
JSONObject info = json.getJSONObject("scheduler");
assertEquals("incorrect number of elements", 1, info.length());
info = info.getJSONObject("schedulerInfo");
assertEquals("incorrect number of elements", 22, info.length());
assertEquals("incorrect number of elements", 23, info.length());
JSONObject capacitiesJsonObject = info.getJSONObject(CAPACITIES);
JSONArray partitionsCapsArray =
capacitiesJsonObject.getJSONArray(QUEUE_CAPACITIES_BY_PARTITION);