diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java index 9794acd34f..a6fcfe6966 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java @@ -46,6 +46,8 @@ public class CapacitySchedulerInfo extends SchedulerInfo { protected float capacity; protected float usedCapacity; protected float maxCapacity; + protected float weight; + protected float normalizedWeight; protected String queueName; protected CapacitySchedulerQueueInfoList queues; protected QueueCapacitiesInfo capacities; @@ -70,6 +72,8 @@ public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs) { if (max < EPSILON || max > 1f) max = 1f; this.maxCapacity = max * 100; + this.weight = parent.getQueueCapacities().getWeight(); + this.normalizedWeight = parent.getQueueCapacities().getNormalizedWeight(); capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(), parent.getQueueResourceQuotas(), false); @@ -147,7 +151,7 @@ protected CapacitySchedulerQueueInfoList getQueues( CapacityScheduler cs, CSQueue parent) { CapacitySchedulerQueueInfoList queuesInfo = new CapacitySchedulerQueueInfoList(); - // JAXB marashalling leads to situation where the "type" field injected + // JAXB marshalling leads to situation where the "type" field injected // for JSON changes from string to array depending on order of printing // Issue gets fixed if all the leaf queues are marshalled before the // non-leaf queues. See YARN-4785 for more details. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index 6fb0290b1d..5e45f53c64 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -66,6 +66,8 @@ public class CapacitySchedulerQueueInfo { protected float absoluteCapacity; protected float absoluteMaxCapacity; protected float absoluteUsedCapacity; + protected float weight; + protected float normalizedWeight; protected int numApplications; protected String queueName; protected boolean isAbsoluteResource; @@ -109,6 +111,8 @@ public class CapacitySchedulerQueueInfo { cap(q.getAbsoluteMaximumCapacity(), 0f, 1f) * 100; absoluteUsedCapacity = cap(q.getAbsoluteUsedCapacity(), 0f, 1f) * 100; + weight = q.getQueueCapacities().getWeight(); + normalizedWeight = q.getQueueCapacities().getNormalizedWeight(); numApplications = q.getNumApplications(); allocatedContainers = q.getMetrics().getAllocatedContainers(); pendingContainers = q.getMetrics().getPendingContainers(); @@ -314,4 +318,12 @@ public LeafQueueTemplateInfo getLeafQueueTemplate() { public String getMode() { return mode; } + + public float getWeight() { + return weight; + } + + public float getNormalizedWeight() { + return normalizedWeight; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java index cc4b565ef3..1b66808356 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java @@ -39,6 +39,8 @@ public class PartitionQueueCapacitiesInfo { private float absoluteUsedCapacity; private float absoluteMaxCapacity = 100; private float maxAMLimitPercentage; + private float weight; + private float normalizedWeight; private ResourceInfo configuredMinResource; private ResourceInfo configuredMaxResource; private ResourceInfo effectiveMinResource; @@ -50,6 +52,7 @@ public PartitionQueueCapacitiesInfo() { public PartitionQueueCapacitiesInfo(String partitionName, float capacity, float usedCapacity, float maxCapacity, float absCapacity, float absUsedCapacity, float absMaxCapacity, float maxAMLimitPercentage, + float weight, float normalizedWeight, Resource confMinRes, Resource confMaxRes, Resource effMinRes, Resource effMaxRes) { super(); @@ -61,6 +64,8 @@ public PartitionQueueCapacitiesInfo(String partitionName, float capacity, this.absoluteUsedCapacity = absUsedCapacity; this.absoluteMaxCapacity = absMaxCapacity; this.maxAMLimitPercentage = maxAMLimitPercentage; + this.weight = weight; + this.normalizedWeight = normalizedWeight; this.configuredMinResource = new ResourceInfo(confMinRes); this.configuredMaxResource = new ResourceInfo(confMaxRes); this.effectiveMinResource = new ResourceInfo(effMinRes); @@ -127,6 +132,22 @@ public float getMaxAMLimitPercentage() { return maxAMLimitPercentage; } + public float getWeight() { + return weight; + } + + public void setWeight(float weight) { + this.weight = weight; + } + + public float getNormalizedWeight() { + return normalizedWeight; + } + + public void setNormalizedWeight(float normalizedWeight) { + this.normalizedWeight = normalizedWeight; + } + public void setMaxAMLimitPercentage(float maxAMLimitPercentage) { this.maxAMLimitPercentage = maxAMLimitPercentage; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java index 35c80d2ea4..3c29f505d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java @@ -52,6 +52,8 @@ public QueueCapacitiesInfo(QueueCapacities capacities, float absUsedCapacity; float absMaxCapacity; float maxAMLimitPercentage; + float weight; + float normalizedWeight; for (String partitionName : capacities.getExistingNodeLabels()) { usedCapacity = capacities.getUsedCapacity(partitionName) * 100; capacity = capacities.getCapacity(partitionName) * 100; @@ -67,10 +69,13 @@ public QueueCapacitiesInfo(QueueCapacities capacities, if (maxCapacity < CapacitySchedulerQueueInfo.EPSILON || maxCapacity > 1f) maxCapacity = 1f; maxCapacity = maxCapacity * 100; + weight = capacities.getWeight(partitionName); + normalizedWeight = capacities.getNormalizedWeight(partitionName); queueCapacitiesByPartition.add(new PartitionQueueCapacitiesInfo( partitionName, capacity, usedCapacity, maxCapacity, absCapacity, absUsedCapacity, absMaxCapacity, considerAMUsage ? maxAMLimitPercentage : 0f, + weight, normalizedWeight, resourceQuotas.getConfiguredMinResource(partitionName), resourceQuotas.getConfiguredMaxResource(partitionName), resourceQuotas.getEffectiveMinResource(partitionName), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index 9fbd2178c1..8699b880d5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -362,7 +362,7 @@ private void verifyClusterScheduler(JSONObject json) throws JSONException, JSONObject info = json.getJSONObject("scheduler"); assertEquals("incorrect number of elements in: " + info, 1, info.length()); info = info.getJSONObject("schedulerInfo"); - assertEquals("incorrect number of elements in: " + info, 13, info.length()); + assertEquals("incorrect number of elements in: " + info, 15, info.length()); verifyClusterSchedulerGeneric(info.getString("type"), (float) info.getDouble("usedCapacity"), (float) info.getDouble("capacity"), @@ -413,10 +413,10 @@ private void verifyClusterSchedulerGeneric(String type, float usedCapacity, private void verifySubQueue(JSONObject info, String q, float parentAbsCapacity, float parentAbsMaxCapacity) throws JSONException, Exception { - int numExpectedElements = 28; + int numExpectedElements = 30; boolean isParentQueue = true; if (!info.has("queues")) { - numExpectedElements = 46; + numExpectedElements = 48; isParentQueue = false; } assertEquals("incorrect number of elements", numExpectedElements, info.length()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java index 8ee00296e6..4a42ffa70a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java @@ -63,9 +63,26 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends JerseyTestBase { private static final Logger LOG = LoggerFactory.getLogger(TestRMWebServicesCapacitySchedDynamicConfig.class); + private static final float EXP_WEIGHT_NON_WEIGHT_MODE = -1.0F; + private static final float EXP_NORM_WEIGHT_NON_WEIGHT_MODE = 0.0F; + private static final float EXP_ROOT_WEIGHT_IN_WEIGHT_MODE = 1.0F; + private static final double DELTA = 0.00001; protected static MockRM rm; + private static class ExpectedQueueWithProperties { + private String path; + public final float weight; + public final float normalizedWeight; + + public ExpectedQueueWithProperties(String path, float weight, + float normalizedWeight) { + this.path = path; + this.weight = weight; + this.normalizedWeight = normalizedWeight; + } + } + private static class WebServletModule extends ServletModule { private final Configuration conf; @@ -124,8 +141,15 @@ public void testSchedulerResponsePercentageMode() initResourceManager(config); JSONObject json = sendRequestToSchedulerEndpoint(); - validateSchedulerInfo(json, "percentage", "root.default", "root.test1", - "root.test2"); + validateSchedulerInfo(json, "percentage", + new ExpectedQueueWithProperties("root", + EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE), + new ExpectedQueueWithProperties("root.default", + EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE), + new ExpectedQueueWithProperties("root.test1", + EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE), + new ExpectedQueueWithProperties("root.test2", + EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE)); } @Test @@ -138,8 +162,15 @@ public void testSchedulerResponseAbsoluteMode() initResourceManager(config); JSONObject json = sendRequestToSchedulerEndpoint(); - validateSchedulerInfo(json, "absolute", "root.default", "root.test1", - "root.test2"); + validateSchedulerInfo(json, "absolute", + new ExpectedQueueWithProperties("root", + EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE), + new ExpectedQueueWithProperties("root.default", + EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE), + new ExpectedQueueWithProperties("root.test1", + EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE), + new ExpectedQueueWithProperties("root.test2", + EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE)); } @Test @@ -152,8 +183,12 @@ public void testSchedulerResponseWeightMode() initResourceManager(config); JSONObject json = sendRequestToSchedulerEndpoint(); - validateSchedulerInfo(json, "weight", "root.default", "root.test1", - "root.test2"); + validateSchedulerInfo(json, "weight", + new ExpectedQueueWithProperties("root", + EXP_ROOT_WEIGHT_IN_WEIGHT_MODE, EXP_ROOT_WEIGHT_IN_WEIGHT_MODE), + new ExpectedQueueWithProperties("root.default", 10.0f, 0.5f), + new ExpectedQueueWithProperties("root.test1", 4.0f, 0.2f), + new ExpectedQueueWithProperties("root.test2", 6.0f, 0.3f)); } private JSONObject sendRequestToSchedulerEndpoint() throws Exception { @@ -169,7 +204,14 @@ private JSONObject sendRequestToSchedulerEndpoint() throws Exception { } private void validateSchedulerInfo(JSONObject json, String expectedMode, - String... expectedQueues) throws JSONException { + ExpectedQueueWithProperties rootQueue, + ExpectedQueueWithProperties... expectedQueues) throws JSONException { + Map queuesMap = new HashMap<>(); + for (ExpectedQueueWithProperties expectedQueue : expectedQueues) { + queuesMap.put(expectedQueue.path, expectedQueue); + } + + int expectedQSize = expectedQueues.length; Assert.assertNotNull("SchedulerTypeInfo should not be null", json); assertEquals("incorrect number of elements in: " + json, 1, json.length()); @@ -178,11 +220,15 @@ private void validateSchedulerInfo(JSONObject json, String expectedMode, Assert.assertNotNull("Scheduler object should not be null", json); assertEquals("incorrect number of elements in: " + info, 1, info.length()); - //Validate if root queue has the expected mode + //Validate if root queue has the expected mode and weight values info = info.getJSONObject("schedulerInfo"); Assert.assertNotNull("SchedulerInfo should not be null", info); Assert.assertEquals("Expected Queue mode " +expectedMode, expectedMode, info.getString("mode")); + Assert.assertEquals(rootQueue.weight, + Float.parseFloat(info.getString("weight")), DELTA); + Assert.assertEquals(rootQueue.normalizedWeight, + Float.parseFloat(info.getString("normalizedWeight")), DELTA); JSONObject queuesObj = info.getJSONObject("queues"); Assert.assertNotNull("QueueInfoList should not be null", queuesObj); @@ -200,10 +246,22 @@ private void validateSchedulerInfo(JSONObject json, String expectedMode, obj.getString("queueName"); String mode = obj.getString("mode"); modesMap.put(queuePath, mode); + + //validate weights of all other queues + ExpectedQueueWithProperties expectedQueue = queuesMap.get(queuePath); + Assert.assertNotNull("Queue not found in expectedQueueMap with path: " + + queuePath, expectedQueue); + Assert.assertEquals("Weight value does not match", + expectedQueue.weight, Float.parseFloat(obj.getString("weight")), + DELTA); + Assert.assertEquals("Normalized weight value does not match", + expectedQueue.normalizedWeight, + Float.parseFloat(obj.getString("normalizedWeight")), DELTA); } //Validate queue paths and modes List sortedExpectedPaths = Arrays.stream(expectedQueues) + .map(eq -> eq.path) .sorted(Comparator.comparing(String::toLowerCase)) .collect(Collectors.toList()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java index 5ce943e5f1..5c74c06429 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java @@ -574,7 +574,7 @@ private void verifySchedulerInfoJson(JSONObject json) JSONObject info = json.getJSONObject("scheduler"); assertEquals("incorrect number of elements", 1, info.length()); info = info.getJSONObject("schedulerInfo"); - assertEquals("incorrect number of elements", 13, info.length()); + assertEquals("incorrect number of elements", 15, info.length()); JSONObject capacitiesJsonObject = info.getJSONObject(CAPACITIES); JSONArray partitionsCapsArray = capacitiesJsonObject.getJSONArray(QUEUE_CAPACITIES_BY_PARTITION);