big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10579. CS Flexible AQC: Modify RM /scheduler endpoint to include weight values for queues. Contributed by Szilard Nemeth

Browse Source
This commit is contained in:
Szilard Nemeth 2021-01-21 09:23:11 +01:00
parent 8bc2dfbf36
commit 06fef5ee43
7 changed files with 113 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
View File

@ -46,6 +46,8 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
protected float capacity;
protected float usedCapacity;
protected float maxCapacity;
protected float weight;
protected float normalizedWeight;
protected String queueName;
protected CapacitySchedulerQueueInfoList queues;
protected QueueCapacitiesInfo capacities;
@ -70,6 +72,8 @@ public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs) {
if (max < EPSILON || max > 1f)
max = 1f;
this.maxCapacity = max * 100;
this.weight = parent.getQueueCapacities().getWeight();
this.normalizedWeight = parent.getQueueCapacities().getNormalizedWeight();
capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(),
parent.getQueueResourceQuotas(), false);
@ -147,7 +151,7 @@ protected CapacitySchedulerQueueInfoList getQueues(
CapacityScheduler cs, CSQueue parent) {
CapacitySchedulerQueueInfoList queuesInfo =
new CapacitySchedulerQueueInfoList();
// JAXB marashalling leads to situation where the "type" field injected
// JAXB marshalling leads to situation where the "type" field injected
// for JSON changes from string to array depending on order of printing
// Issue gets fixed if all the leaf queues are marshalled before the
// non-leaf queues. See YARN-4785 for more details.

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
View File

@ -66,6 +66,8 @@ public class CapacitySchedulerQueueInfo {
protected float absoluteCapacity;
protected float absoluteMaxCapacity;
protected float absoluteUsedCapacity;
protected float weight;
protected float normalizedWeight;
protected int numApplications;
protected String queueName;
protected boolean isAbsoluteResource;
@ -109,6 +111,8 @@ public class CapacitySchedulerQueueInfo {
cap(q.getAbsoluteMaximumCapacity(), 0f, 1f) * 100;
absoluteUsedCapacity =
cap(q.getAbsoluteUsedCapacity(), 0f, 1f) * 100;
weight = q.getQueueCapacities().getWeight();
normalizedWeight = q.getQueueCapacities().getNormalizedWeight();
numApplications = q.getNumApplications();
allocatedContainers = q.getMetrics().getAllocatedContainers();
pendingContainers = q.getMetrics().getPendingContainers();
@ -314,4 +318,12 @@ public LeafQueueTemplateInfo getLeafQueueTemplate() {
public String getMode() {
return mode;
}
public float getWeight() {
return weight;
}
public float getNormalizedWeight() {
return normalizedWeight;
}
}

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java
View File

@ -39,6 +39,8 @@ public class PartitionQueueCapacitiesInfo {
private float absoluteUsedCapacity;
private float absoluteMaxCapacity = 100;
private float maxAMLimitPercentage;
private float weight;
private float normalizedWeight;
private ResourceInfo configuredMinResource;
private ResourceInfo configuredMaxResource;
private ResourceInfo effectiveMinResource;
@ -50,6 +52,7 @@ public PartitionQueueCapacitiesInfo() {
public PartitionQueueCapacitiesInfo(String partitionName, float capacity,
float usedCapacity, float maxCapacity, float absCapacity,
float absUsedCapacity, float absMaxCapacity, float maxAMLimitPercentage,
float weight, float normalizedWeight,
Resource confMinRes, Resource confMaxRes, Resource effMinRes,
Resource effMaxRes) {
super();
@ -61,6 +64,8 @@ public PartitionQueueCapacitiesInfo(String partitionName, float capacity,
this.absoluteUsedCapacity = absUsedCapacity;
this.absoluteMaxCapacity = absMaxCapacity;
this.maxAMLimitPercentage = maxAMLimitPercentage;
this.weight = weight;
this.normalizedWeight = normalizedWeight;
this.configuredMinResource = new ResourceInfo(confMinRes);
this.configuredMaxResource = new ResourceInfo(confMaxRes);
this.effectiveMinResource = new ResourceInfo(effMinRes);
@ -127,6 +132,22 @@ public float getMaxAMLimitPercentage() {
return maxAMLimitPercentage;
}
public float getWeight() {
return weight;
}
public void setWeight(float weight) {
this.weight = weight;
}
public float getNormalizedWeight() {
return normalizedWeight;
}
public void setNormalizedWeight(float normalizedWeight) {
this.normalizedWeight = normalizedWeight;
}
public void setMaxAMLimitPercentage(float maxAMLimitPercentage) {
this.maxAMLimitPercentage = maxAMLimitPercentage;
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java
View File

@ -52,6 +52,8 @@ public QueueCapacitiesInfo(QueueCapacities capacities,
float absUsedCapacity;
float absMaxCapacity;
float maxAMLimitPercentage;
float weight;
float normalizedWeight;
for (String partitionName : capacities.getExistingNodeLabels()) {
usedCapacity = capacities.getUsedCapacity(partitionName) * 100;
capacity = capacities.getCapacity(partitionName) * 100;
@ -67,10 +69,13 @@ public QueueCapacitiesInfo(QueueCapacities capacities,
if (maxCapacity < CapacitySchedulerQueueInfo.EPSILON || maxCapacity > 1f)
maxCapacity = 1f;
maxCapacity = maxCapacity * 100;
weight = capacities.getWeight(partitionName);
normalizedWeight = capacities.getNormalizedWeight(partitionName);
queueCapacitiesByPartition.add(new PartitionQueueCapacitiesInfo(
partitionName, capacity, usedCapacity, maxCapacity, absCapacity,
absUsedCapacity, absMaxCapacity,
considerAMUsage ? maxAMLimitPercentage : 0f,
weight, normalizedWeight,
resourceQuotas.getConfiguredMinResource(partitionName),
resourceQuotas.getConfiguredMaxResource(partitionName),
resourceQuotas.getEffectiveMinResource(partitionName),

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
View File

@ -362,7 +362,7 @@ private void verifyClusterScheduler(JSONObject json) throws JSONException,
JSONObject info = json.getJSONObject("scheduler");
assertEquals("incorrect number of elements in: " + info, 1, info.length());
info = info.getJSONObject("schedulerInfo");
assertEquals("incorrect number of elements in: " + info, 13, info.length());
assertEquals("incorrect number of elements in: " + info, 15, info.length());
verifyClusterSchedulerGeneric(info.getString("type"),
(float) info.getDouble("usedCapacity"),
(float) info.getDouble("capacity"),
@ -413,10 +413,10 @@ private void verifyClusterSchedulerGeneric(String type, float usedCapacity,
private void verifySubQueue(JSONObject info, String q,
float parentAbsCapacity, float parentAbsMaxCapacity)
throws JSONException, Exception {
int numExpectedElements = 28;
int numExpectedElements = 30;
boolean isParentQueue = true;
if (!info.has("queues")) {
numExpectedElements = 46;
numExpectedElements = 48;
isParentQueue = false;
}
assertEquals("incorrect number of elements", numExpectedElements, info.length());

74
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java
View File

@ -63,9 +63,26 @@ public class TestRMWebServicesCapacitySchedDynamicConfig extends
JerseyTestBase {
private static final Logger LOG =
LoggerFactory.getLogger(TestRMWebServicesCapacitySchedDynamicConfig.class);
private static final float EXP_WEIGHT_NON_WEIGHT_MODE = -1.0F;
private static final float EXP_NORM_WEIGHT_NON_WEIGHT_MODE = 0.0F;
private static final float EXP_ROOT_WEIGHT_IN_WEIGHT_MODE = 1.0F;
private static final double DELTA = 0.00001;
protected static MockRM rm;
private static class ExpectedQueueWithProperties {
private String path;
public final float weight;
public final float normalizedWeight;
public ExpectedQueueWithProperties(String path, float weight,
float normalizedWeight) {
this.path = path;
this.weight = weight;
this.normalizedWeight = normalizedWeight;
}
}
private static class WebServletModule extends ServletModule {
private final Configuration conf;
@ -124,8 +141,15 @@ public void testSchedulerResponsePercentageMode()
initResourceManager(config);
JSONObject json = sendRequestToSchedulerEndpoint();
validateSchedulerInfo(json, "percentage", "root.default", "root.test1",
"root.test2");
validateSchedulerInfo(json, "percentage",
new ExpectedQueueWithProperties("root",
EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE),
new ExpectedQueueWithProperties("root.default",
EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE),
new ExpectedQueueWithProperties("root.test1",
EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE),
new ExpectedQueueWithProperties("root.test2",
EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE));
}
@Test
@ -138,8 +162,15 @@ public void testSchedulerResponseAbsoluteMode()
initResourceManager(config);
JSONObject json = sendRequestToSchedulerEndpoint();
validateSchedulerInfo(json, "absolute", "root.default", "root.test1",
"root.test2");
validateSchedulerInfo(json, "absolute",
new ExpectedQueueWithProperties("root",
EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE),
new ExpectedQueueWithProperties("root.default",
EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE),
new ExpectedQueueWithProperties("root.test1",
EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE),
new ExpectedQueueWithProperties("root.test2",
EXP_WEIGHT_NON_WEIGHT_MODE, EXP_NORM_WEIGHT_NON_WEIGHT_MODE));
}
@Test
@ -152,8 +183,12 @@ public void testSchedulerResponseWeightMode()
initResourceManager(config);
JSONObject json = sendRequestToSchedulerEndpoint();
validateSchedulerInfo(json, "weight", "root.default", "root.test1",
"root.test2");
validateSchedulerInfo(json, "weight",
new ExpectedQueueWithProperties("root",
EXP_ROOT_WEIGHT_IN_WEIGHT_MODE, EXP_ROOT_WEIGHT_IN_WEIGHT_MODE),
new ExpectedQueueWithProperties("root.default", 10.0f, 0.5f),
new ExpectedQueueWithProperties("root.test1", 4.0f, 0.2f),
new ExpectedQueueWithProperties("root.test2", 6.0f, 0.3f));
}
private JSONObject sendRequestToSchedulerEndpoint() throws Exception {
@ -169,7 +204,14 @@ private JSONObject sendRequestToSchedulerEndpoint() throws Exception {
}
private void validateSchedulerInfo(JSONObject json, String expectedMode,
String... expectedQueues) throws JSONException {
ExpectedQueueWithProperties rootQueue,
ExpectedQueueWithProperties... expectedQueues) throws JSONException {
Map<String, ExpectedQueueWithProperties> queuesMap = new HashMap<>();
for (ExpectedQueueWithProperties expectedQueue : expectedQueues) {
queuesMap.put(expectedQueue.path, expectedQueue);
}
int expectedQSize = expectedQueues.length;
Assert.assertNotNull("SchedulerTypeInfo should not be null", json);
assertEquals("incorrect number of elements in: " + json, 1, json.length());
@ -178,11 +220,15 @@ private void validateSchedulerInfo(JSONObject json, String expectedMode,
Assert.assertNotNull("Scheduler object should not be null", json);
assertEquals("incorrect number of elements in: " + info, 1, info.length());
//Validate if root queue has the expected mode
//Validate if root queue has the expected mode and weight values
info = info.getJSONObject("schedulerInfo");
Assert.assertNotNull("SchedulerInfo should not be null", info);
Assert.assertEquals("Expected Queue mode " +expectedMode, expectedMode,
info.getString("mode"));
Assert.assertEquals(rootQueue.weight,
Float.parseFloat(info.getString("weight")), DELTA);
Assert.assertEquals(rootQueue.normalizedWeight,
Float.parseFloat(info.getString("normalizedWeight")), DELTA);
JSONObject queuesObj = info.getJSONObject("queues");
Assert.assertNotNull("QueueInfoList should not be null", queuesObj);
@ -200,10 +246,22 @@ private void validateSchedulerInfo(JSONObject json, String expectedMode,
obj.getString("queueName");
String mode = obj.getString("mode");
modesMap.put(queuePath, mode);
//validate weights of all other queues
ExpectedQueueWithProperties expectedQueue = queuesMap.get(queuePath);
Assert.assertNotNull("Queue not found in expectedQueueMap with path: " +
queuePath, expectedQueue);
Assert.assertEquals("Weight value does not match",
expectedQueue.weight, Float.parseFloat(obj.getString("weight")),
DELTA);
Assert.assertEquals("Normalized weight value does not match",
expectedQueue.normalizedWeight,
Float.parseFloat(obj.getString("normalizedWeight")), DELTA);
}
//Validate queue paths and modes
List<String> sortedExpectedPaths = Arrays.stream(expectedQueues)
.map(eq -> eq.path)
.sorted(Comparator.comparing(String::toLowerCase))
.collect(Collectors.toList());

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java
View File

@ -574,7 +574,7 @@ private void verifySchedulerInfoJson(JSONObject json)
JSONObject info = json.getJSONObject("scheduler");
assertEquals("incorrect number of elements", 1, info.length());
info = info.getJSONObject("schedulerInfo");
assertEquals("incorrect number of elements", 13, info.length());
assertEquals("incorrect number of elements", 15, info.length());
JSONObject capacitiesJsonObject = info.getJSONObject(CAPACITIES);
JSONArray partitionsCapsArray =
capacitiesJsonObject.getJSONArray(QUEUE_CAPACITIES_BY_PARTITION);