big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10807. Parents node labels are incorrectly added to child queues in weight mode. Contributed by Benjamin Teke.

Browse Source
This commit is contained in:
zhuqi-lucas 2021-06-08 21:03:43 +08:00
parent c748fce17a
commit ec16b1d3b9
2 changed files with 42 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
View File

@ -1250,16 +1250,23 @@ public void updateClusterResource(Resource clusterResource,
float sumOfWeight = 0;
for (CSQueue queue : childQueues) {
float weight = Math.max(0,
queue.getQueueCapacities().getWeight(nodeLabel));
sumOfWeight += weight;
if (queue.getQueueCapacities().getExistingNodeLabels()
.contains(nodeLabel)) {
float weight = Math.max(0,
queue.getQueueCapacities().getWeight(nodeLabel));
sumOfWeight += weight;
}
}
// When sum of weight == 0, skip setting normalized_weight (so
// normalized weight will be 0).
if (Math.abs(sumOfWeight) > 1e-6) {
for (CSQueue queue : childQueues) {
queue.getQueueCapacities().setNormalizedWeight(nodeLabel,
queue.getQueueCapacities().getWeight(nodeLabel) / sumOfWeight);
if (queue.getQueueCapacities().getExistingNodeLabels()
.contains(nodeLabel)) {
queue.getQueueCapacities().setNormalizedWeight(nodeLabel,
queue.getQueueCapacities().getWeight(nodeLabel) /
sumOfWeight);
}
}
}
}

30
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java
View File

@ -338,6 +338,36 @@ public void testGetCapacityOrWeightStringParentPctLeafWeights()
}
}
/**
* This test ensures that while iterating through a parent's Node Labels
* (when calculating the normalized weights) the parent's Node Labels won't
* be added to the children with weight -1. If the parent
* has a node label that a specific child doesn't the normalized calling the
* normalized weight setter will be skipped. The queue root.b has access to
* the labels "x" and "y", but root.b.b1 won't. For more information see
* YARN-10807.
* @throws Exception
*/
@Test
public void testChildAccessibleNodeLabelsWeightMode() throws Exception {
MockRM rm = new MockRM(getCSConfWithQueueLabelsWeightOnly(conf));
rm.start();
CapacityScheduler cs =
(CapacityScheduler) rm.getRMContext().getScheduler();
LeafQueue b1 = (LeafQueue) cs.getQueue(B1);
Assert.assertNotNull(b1);
Assert.assertTrue(b1.getAccessibleNodeLabels().isEmpty());
Set<String> b1ExistingNodeLabels = ((CSQueue) b1).getQueueCapacities()
.getExistingNodeLabels();
Assert.assertEquals(1, b1ExistingNodeLabels.size());
Assert.assertEquals("", b1ExistingNodeLabels.iterator().next());
rm.close();
}
@Test
public void testQueueInfoWeight() throws Exception {
MockRM rm = new MockRM(conf);