big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-11023. Extend the root QueueInfo with max-parallel-apps in CapacityScheduler. Contributed by Tamas Domok

Browse Source
This commit is contained in:
Szilard Nemeth 2021-12-07 15:26:07 +01:00
parent 79cc7b46af
commit 60440a836a
3 changed files with 10 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
View File

@ -50,6 +50,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
protected float normalizedWeight; protected float normalizedWeight;
protected String queueName; protected String queueName;
private String queuePath; private String queuePath;
protected int maxParallelApps;
protected CapacitySchedulerQueueInfoList queues; protected CapacitySchedulerQueueInfoList queues;
protected QueueCapacitiesInfo capacities; protected QueueCapacitiesInfo capacities;
protected CapacitySchedulerHealthInfo health; protected CapacitySchedulerHealthInfo health;
@ -83,6 +84,7 @@ public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs) {
this.maxCapacity = max * 100; this.maxCapacity = max * 100;
this.weight = parent.getQueueCapacities().getWeight(); this.weight = parent.getQueueCapacities().getWeight();
this.normalizedWeight = parent.getQueueCapacities().getNormalizedWeight(); this.normalizedWeight = parent.getQueueCapacities().getNormalizedWeight();
this.maxParallelApps = parent.getMaxParallelApps();
capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(), capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(),
parent.getQueueResourceQuotas(), false); parent.getQueueResourceQuotas(), false);

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
View File

@ -270,7 +270,8 @@ public void verifyClusterSchedulerXML(NodeList nodes) throws Exception {
WebServicesTestUtils.getXmlFloat(element, "capacity"), WebServicesTestUtils.getXmlFloat(element, "capacity"),
WebServicesTestUtils.getXmlFloat(element, "maxCapacity"), WebServicesTestUtils.getXmlFloat(element, "maxCapacity"),
WebServicesTestUtils.getXmlString(element, "queueName"), WebServicesTestUtils.getXmlString(element, "queueName"),
WebServicesTestUtils.getXmlString(element, "queuePath")); WebServicesTestUtils.getXmlString(element, "queuePath"),
WebServicesTestUtils.getXmlInt(element, "maxParallelApps"));
NodeList children = element.getChildNodes(); NodeList children = element.getChildNodes();
for (int j = 0; j < children.getLength(); j++) { for (int j = 0; j < children.getLength(); j++) {
@ -374,13 +375,14 @@ private void verifyClusterScheduler(JSONObject json) throws JSONException,
JSONObject info = json.getJSONObject("scheduler"); JSONObject info = json.getJSONObject("scheduler");
assertEquals("incorrect number of elements in: " + info, 1, info.length()); assertEquals("incorrect number of elements in: " + info, 1, info.length());
info = info.getJSONObject("schedulerInfo"); info = info.getJSONObject("schedulerInfo");
assertEquals("incorrect number of elements in: " + info, 22, info.length()); assertEquals("incorrect number of elements in: " + info, 23, info.length());
verifyClusterSchedulerGeneric(info.getString("type"), verifyClusterSchedulerGeneric(info.getString("type"),
(float) info.getDouble("usedCapacity"), (float) info.getDouble("usedCapacity"),
(float) info.getDouble("capacity"), (float) info.getDouble("capacity"),
(float) info.getDouble("maxCapacity"), (float) info.getDouble("maxCapacity"),
info.getString("queueName"), info.getString("queueName"),
info.getString("queuePath")); info.getString("queuePath"),
info.getInt("maxParallelApps"));
JSONObject health = info.getJSONObject("health"); JSONObject health = info.getJSONObject("health");
assertNotNull(health); assertNotNull(health);
assertEquals("incorrect number of elements in: " + health, 3, assertEquals("incorrect number of elements in: " + health, 3,
@ -415,7 +417,7 @@ private void verifyClusterScheduler(JSONObject json) throws JSONException,
} }
private void verifyClusterSchedulerGeneric(String type, float usedCapacity, private void verifyClusterSchedulerGeneric(String type, float usedCapacity,
float capacity, float maxCapacity, String queueName, String queuePath) float capacity, float maxCapacity, String queueName, String queuePath, int maxParallelApps)
throws Exception { throws Exception {
assertTrue("type doesn't match", "capacityScheduler".matches(type)); assertTrue("type doesn't match", "capacityScheduler".matches(type));
@ -424,6 +426,7 @@ private void verifyClusterSchedulerGeneric(String type, float usedCapacity,
assertEquals("maxCapacity doesn't match", 100, maxCapacity, 1e-3f); assertEquals("maxCapacity doesn't match", 100, maxCapacity, 1e-3f);
assertTrue("queueName doesn't match", "root".matches(queueName)); assertTrue("queueName doesn't match", "root".matches(queueName));
assertTrue("queuePath doesn't match", "root".matches(queuePath)); assertTrue("queuePath doesn't match", "root".matches(queuePath));
assertEquals("maxParallelApps doesn't match ", Integer.MAX_VALUE, maxParallelApps);
} }
private void verifySubQueue(JSONObject info, String q, private void verifySubQueue(JSONObject info, String q,

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java
View File

@ -574,7 +574,7 @@ private void verifySchedulerInfoJson(JSONObject json)
JSONObject info = json.getJSONObject("scheduler"); JSONObject info = json.getJSONObject("scheduler");
assertEquals("incorrect number of elements", 1, info.length()); assertEquals("incorrect number of elements", 1, info.length());
info = info.getJSONObject("schedulerInfo"); info = info.getJSONObject("schedulerInfo");
assertEquals("incorrect number of elements", 22, info.length()); assertEquals("incorrect number of elements", 23, info.length());
JSONObject capacitiesJsonObject = info.getJSONObject(CAPACITIES); JSONObject capacitiesJsonObject = info.getJSONObject(CAPACITIES);
JSONArray partitionsCapsArray = JSONArray partitionsCapsArray =
capacitiesJsonObject.getJSONArray(QUEUE_CAPACITIES_BY_PARTITION); capacitiesJsonObject.getJSONArray(QUEUE_CAPACITIES_BY_PARTITION);