big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-8917. Absolute (maximum) capacity of level3+ queues is wrongly calculated for absolute resource. Contributed by Tao Yang.

Browse Source
This commit is contained in:
Rohith Sharma K S 2019-08-26 21:06:15 +05:30
parent 23e532d739
commit 689d2e6105
2 changed files with 53 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
View File

@ -1102,14 +1102,14 @@ private void deriveCapacityFromAbsoluteConfigurations(String label,
// 3. Update absolute capacity as a float based on parent's minResource and // 3. Update absolute capacity as a float based on parent's minResource and
// cluster resource. // cluster resource.
childQueue.getQueueCapacities().setAbsoluteCapacity(label, childQueue.getQueueCapacities().setAbsoluteCapacity(label,
(float) childQueue.getQueueCapacities().getCapacity() childQueue.getQueueCapacities().getCapacity(label)
/ getQueueCapacities().getAbsoluteCapacity(label)); * getQueueCapacities().getAbsoluteCapacity(label));
// 4. Update absolute max-capacity as a float based on parent's maxResource // 4. Update absolute max-capacity as a float based on parent's maxResource
// and cluster resource. // and cluster resource.
childQueue.getQueueCapacities().setAbsoluteMaximumCapacity(label, childQueue.getQueueCapacities().setAbsoluteMaximumCapacity(label,
(float) childQueue.getQueueCapacities().getMaximumCapacity(label) childQueue.getQueueCapacities().getMaximumCapacity(label)
/ getQueueCapacities().getAbsoluteMaximumCapacity(label)); * getQueueCapacities().getAbsoluteMaximumCapacity(label));
// Re-visit max applications for a queue based on absolute capacity if // Re-visit max applications for a queue based on absolute capacity if
// needed. // needed.

49
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
View File

@ -33,6 +33,7 @@
public class TestAbsoluteResourceConfiguration { public class TestAbsoluteResourceConfiguration {
private static final int GB = 1024; private static final int GB = 1024;
private static final float DELTA = 0.001f;
private static final String QUEUEA = "queueA"; private static final String QUEUEA = "queueA";
private static final String QUEUEB = "queueB"; private static final String QUEUEB = "queueB";
@ -544,6 +545,10 @@ public void testEffectiveResourceAfterIncreasingClusterResource()
QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource()); QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource());
Assert.assertEquals("Effective Max resource for QUEUEA is not correct", Assert.assertEquals("Effective Max resource for QUEUEA is not correct",
QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource()); QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource());
Assert.assertEquals("Absolute capacity for QUEUEA is not correct",
0.4, qA.getAbsoluteCapacity(), DELTA);
Assert.assertEquals("Absolute Max capacity for QUEUEA is not correct",
0.8, qA.getAbsoluteMaximumCapacity(), DELTA);
ParentQueue qB = (ParentQueue) cs.getQueue(QUEUEB); ParentQueue qB = (ParentQueue) cs.getQueue(QUEUEB);
Assert.assertNotNull(qB); Assert.assertNotNull(qB);
@ -555,6 +560,10 @@ public void testEffectiveResourceAfterIncreasingClusterResource()
QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource()); QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource());
Assert.assertEquals("Effective Max resource for QUEUEB is not correct", Assert.assertEquals("Effective Max resource for QUEUEB is not correct",
QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource()); QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource());
Assert.assertEquals("Absolute capacity for QUEUEB is not correct",
0.2, qB.getAbsoluteCapacity(), DELTA);
Assert.assertEquals("Absolute Max capacity for QUEUEB is not correct",
0.6, qB.getAbsoluteMaximumCapacity(), DELTA);
LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC); LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC);
Assert.assertNotNull(qC); Assert.assertNotNull(qC);
@ -566,18 +575,30 @@ public void testEffectiveResourceAfterIncreasingClusterResource()
QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource()); QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource());
Assert.assertEquals("Effective Max resource for QUEUEC is not correct", Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource()); QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource());
Assert.assertEquals("Absolute capacity for QUEUEC is not correct",
0.2, qC.getAbsoluteCapacity(), DELTA);
Assert.assertEquals("Absolute Max capacity for QUEUEC is not correct",
0.6, qC.getAbsoluteMaximumCapacity(), DELTA);
LeafQueue qA1 = (LeafQueue) cs.getQueue(QUEUEA1); LeafQueue qA1 = (LeafQueue) cs.getQueue(QUEUEA1);
Assert.assertEquals("Effective Min resource for QUEUEA1 is not correct", Assert.assertEquals("Effective Min resource for QUEUEA1 is not correct",
QUEUE_A1_MINRES, qA1.queueResourceQuotas.getEffectiveMinResource()); QUEUE_A1_MINRES, qA1.queueResourceQuotas.getEffectiveMinResource());
Assert.assertEquals("Effective Max resource for QUEUEA1 is not correct", Assert.assertEquals("Effective Max resource for QUEUEA1 is not correct",
QUEUE_A_MAXRES, qA1.queueResourceQuotas.getEffectiveMaxResource()); QUEUE_A_MAXRES, qA1.queueResourceQuotas.getEffectiveMaxResource());
Assert.assertEquals("Absolute capacity for QUEUEA1 is not correct",
0.2, qA1.getAbsoluteCapacity(), DELTA);
Assert.assertEquals("Absolute Max capacity for QUEUEA1 is not correct",
0.8, qA1.getAbsoluteMaximumCapacity(), DELTA);
LeafQueue qA2 = (LeafQueue) cs.getQueue(QUEUEA2); LeafQueue qA2 = (LeafQueue) cs.getQueue(QUEUEA2);
Assert.assertEquals("Effective Min resource for QUEUEA2 is not correct", Assert.assertEquals("Effective Min resource for QUEUEA2 is not correct",
QUEUE_A2_MINRES, qA2.queueResourceQuotas.getEffectiveMinResource()); QUEUE_A2_MINRES, qA2.queueResourceQuotas.getEffectiveMinResource());
Assert.assertEquals("Effective Max resource for QUEUEA2 is not correct", Assert.assertEquals("Effective Max resource for QUEUEA2 is not correct",
QUEUE_A_MAXRES, qA2.queueResourceQuotas.getEffectiveMaxResource()); QUEUE_A_MAXRES, qA2.queueResourceQuotas.getEffectiveMaxResource());
Assert.assertEquals("Absolute capacity for QUEUEA2 is not correct",
0.2, qA2.getAbsoluteCapacity(), DELTA);
Assert.assertEquals("Absolute Max capacity for QUEUEA2 is not correct",
0.8, qA2.getAbsoluteMaximumCapacity(), DELTA);
LeafQueue qB1 = (LeafQueue) cs.getQueue(QUEUEB1); LeafQueue qB1 = (LeafQueue) cs.getQueue(QUEUEB1);
Assert.assertEquals("Min resource configured for QUEUEB1 is not correct", Assert.assertEquals("Min resource configured for QUEUEB1 is not correct",
@ -588,6 +609,10 @@ public void testEffectiveResourceAfterIncreasingClusterResource()
QUEUE_B1_MINRES, qB1.queueResourceQuotas.getEffectiveMinResource()); QUEUE_B1_MINRES, qB1.queueResourceQuotas.getEffectiveMinResource());
Assert.assertEquals("Effective Max resource for QUEUEB1 is not correct", Assert.assertEquals("Effective Max resource for QUEUEB1 is not correct",
QUEUE_B_MAXRES, qB1.queueResourceQuotas.getEffectiveMaxResource()); QUEUE_B_MAXRES, qB1.queueResourceQuotas.getEffectiveMaxResource());
Assert.assertEquals("Absolute capacity for QUEUEB1 is not correct",
0.16, qB1.getAbsoluteCapacity(), DELTA);
Assert.assertEquals("Absolute Max capacity for QUEUEB1 is not correct",
0.6, qB1.getAbsoluteMaximumCapacity(), DELTA);
// add new NM. // add new NM.
rm.registerNode("127.0.0.3:1234", 125 * GB, 20); rm.registerNode("127.0.0.3:1234", 125 * GB, 20);
@ -599,31 +624,55 @@ public void testEffectiveResourceAfterIncreasingClusterResource()
QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource()); QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource());
Assert.assertEquals("Effective Max resource for QUEUEA is not correct", Assert.assertEquals("Effective Max resource for QUEUEA is not correct",
QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource()); QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource());
Assert.assertEquals("Absolute capacity for QUEUEA is not correct",
0.266, qA.getAbsoluteCapacity(), DELTA);
Assert.assertEquals("Absolute Max capacity for QUEUEA is not correct",
0.533, qA.getAbsoluteMaximumCapacity(), DELTA);
Assert.assertEquals("Effective Min resource for QUEUEB is not correct", Assert.assertEquals("Effective Min resource for QUEUEB is not correct",
QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource()); QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource());
Assert.assertEquals("Effective Max resource for QUEUEB is not correct", Assert.assertEquals("Effective Max resource for QUEUEB is not correct",
QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource()); QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource());
Assert.assertEquals("Absolute capacity for QUEUEB is not correct",
0.133, qB.getAbsoluteCapacity(), DELTA);
Assert.assertEquals("Absolute Max capacity for QUEUEB is not correct",
0.4, qB.getAbsoluteMaximumCapacity(), DELTA);
Assert.assertEquals("Effective Min resource for QUEUEC is not correct", Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource()); QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource());
Assert.assertEquals("Effective Max resource for QUEUEC is not correct", Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource()); QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource());
Assert.assertEquals("Absolute capacity for QUEUEC is not correct",
0.133, qC.getAbsoluteCapacity(), DELTA);
Assert.assertEquals("Absolute Max capacity for QUEUEC is not correct",
0.4, qC.getAbsoluteMaximumCapacity(), DELTA);
Assert.assertEquals("Effective Min resource for QUEUEB1 is not correct", Assert.assertEquals("Effective Min resource for QUEUEB1 is not correct",
QUEUE_B1_MINRES, qB1.queueResourceQuotas.getEffectiveMinResource()); QUEUE_B1_MINRES, qB1.queueResourceQuotas.getEffectiveMinResource());
Assert.assertEquals("Effective Max resource for QUEUEB1 is not correct", Assert.assertEquals("Effective Max resource for QUEUEB1 is not correct",
QUEUE_B_MAXRES, qB1.queueResourceQuotas.getEffectiveMaxResource()); QUEUE_B_MAXRES, qB1.queueResourceQuotas.getEffectiveMaxResource());
Assert.assertEquals("Absolute capacity for QUEUEB1 is not correct",
0.106, qB1.getAbsoluteCapacity(), DELTA);
Assert.assertEquals("Absolute Max capacity for QUEUEB1 is not correct",
0.4, qB1.getAbsoluteMaximumCapacity(), DELTA);
Assert.assertEquals("Effective Min resource for QUEUEA1 is not correct", Assert.assertEquals("Effective Min resource for QUEUEA1 is not correct",
QUEUE_A1_MINRES, qA1.queueResourceQuotas.getEffectiveMinResource()); QUEUE_A1_MINRES, qA1.queueResourceQuotas.getEffectiveMinResource());
Assert.assertEquals("Effective Max resource for QUEUEA1 is not correct", Assert.assertEquals("Effective Max resource for QUEUEA1 is not correct",
QUEUE_A_MAXRES, qA1.queueResourceQuotas.getEffectiveMaxResource()); QUEUE_A_MAXRES, qA1.queueResourceQuotas.getEffectiveMaxResource());
Assert.assertEquals("Absolute capacity for QUEUEA1 is not correct",
0.133, qA1.getAbsoluteCapacity(), DELTA);
Assert.assertEquals("Absolute Max capacity for QUEUEA1 is not correct",
0.533, qA1.getAbsoluteMaximumCapacity(), DELTA);
Assert.assertEquals("Effective Min resource for QUEUEA2 is not correct", Assert.assertEquals("Effective Min resource for QUEUEA2 is not correct",
QUEUE_A2_MINRES, qA2.queueResourceQuotas.getEffectiveMinResource()); QUEUE_A2_MINRES, qA2.queueResourceQuotas.getEffectiveMinResource());
Assert.assertEquals("Effective Max resource for QUEUEA2 is not correct", Assert.assertEquals("Effective Max resource for QUEUEA2 is not correct",
QUEUE_A_MAXRES, qA2.queueResourceQuotas.getEffectiveMaxResource()); QUEUE_A_MAXRES, qA2.queueResourceQuotas.getEffectiveMaxResource());
Assert.assertEquals("Absolute capacity for QUEUEA2 is not correct",
0.133, qA2.getAbsoluteCapacity(), DELTA);
Assert.assertEquals("Absolute Max capacity for QUEUEA2 is not correct",
0.533, qA2.getAbsoluteMaximumCapacity(), DELTA);
rm.stop(); rm.stop();
} }