From 4f008153ef5fca9e1f71ebc7069c502e803ab1e8 Mon Sep 17 00:00:00 2001 From: Wangda Tan Date: Thu, 21 Jan 2021 09:31:14 -0800 Subject: [PATCH] YARN-10587. Fix AutoCreateLeafQueueCreation cap related caculation when in absolute mode. (Qi Zhu via wangda) Change-Id: I8889c393a9f08b633c72a2e873c6f45ca12cc82b --- .../GuaranteedOrZeroCapacityOverTimePolicy.java | 9 ++++++--- .../capacity/TestAbsoluteResourceConfiguration.java | 12 +++++++----- .../capacity/TestAbsoluteResourceWithAutoQueue.java | 12 ++++++------ 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java index ab99317888..a53fe4a391 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java @@ -19,6 +19,7 @@ .queuemanagement; import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity .QueueManagementDynamicEditPolicy; @@ -624,7 +625,7 @@ private Map deactivateLeafQueuesIfInActive( QueueCapacities capacities = leafQueueEntitlements.get( leafQueue.getQueuePath()); - updateToZeroCapacity(capacities, nodeLabel); + updateToZeroCapacity(capacities, nodeLabel, (LeafQueue)childQueue); deactivatedQueues.put(leafQueue.getQueuePath(), leafQueueTemplateCapacities); } @@ -822,7 +823,7 @@ public AutoCreatedLeafQueueConfig getInitialLeafQueueConfiguration( updateCapacityFromTemplate(capacities, nodeLabel); activate(leafQueue, nodeLabel); } else{ - updateToZeroCapacity(capacities, nodeLabel); + updateToZeroCapacity(capacities, nodeLabel, leafQueue); } } @@ -834,10 +835,12 @@ public AutoCreatedLeafQueueConfig getInitialLeafQueueConfiguration( } private void updateToZeroCapacity(QueueCapacities capacities, - String nodeLabel) { + String nodeLabel, LeafQueue leafQueue) { capacities.setCapacity(nodeLabel, 0.0f); capacities.setMaximumCapacity(nodeLabel, leafQueueTemplateCapacities.getMaximumCapacity(nodeLabel)); + leafQueue.getQueueResourceQuotas(). + setConfiguredMinResource(nodeLabel, Resource.newInstance(0, 0)); } private void updateCapacityFromTemplate(QueueCapacities capacities, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java index da13e18cfc..97f9a652c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java @@ -231,8 +231,10 @@ public void testSimpleMinMaxResourceConfigurartionPerQueue() d1.queueResourceQuotas.getEffectiveMaxResource()); /** - * After adding child queue d2, adjustment happens and both d1 and d2 shares - * resource of Managed Parent Queue + * After adding child queue d2, d1 + d2 > resource + * of Managed Parent queue, d2 will change to 0. + * d1 will occupy all entire resource + * of Managed Parent queue. */ AutoCreatedLeafQueue d2 = new AutoCreatedLeafQueue(cs, "d2", parentQueue); cs.addQueue(d2); @@ -240,9 +242,9 @@ public void testSimpleMinMaxResourceConfigurartionPerQueue() cs.getRootQueue().updateClusterResource(cs.getClusterResource(), new ResourceLimits(cs.getClusterResource())); - Assert.assertEquals(QUEUE_D_TEMPL_MINRES, + Assert.assertEquals(Resource.newInstance(0, 0), d2.queueResourceQuotas.getConfiguredMinResource()); - Assert.assertEquals(Resource.newInstance(12800, 2), + Assert.assertEquals(Resource.newInstance(0, 0), d2.queueResourceQuotas.getEffectiveMinResource()); Assert.assertEquals(QUEUE_D_TEMPL_MAXRES, d2.queueResourceQuotas.getConfiguredMaxResource()); @@ -251,7 +253,7 @@ public void testSimpleMinMaxResourceConfigurartionPerQueue() Assert.assertEquals(QUEUE_D_TEMPL_MINRES, d1.queueResourceQuotas.getConfiguredMinResource()); - Assert.assertEquals(Resource.newInstance(12800, 2), + Assert.assertEquals(QUEUE_D_TEMPL_MINRES, d1.queueResourceQuotas.getEffectiveMinResource()); Assert.assertEquals(QUEUE_D_TEMPL_MAXRES, d1.queueResourceQuotas.getConfiguredMaxResource()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java index f9b494ece8..2bad8b7447 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java @@ -235,12 +235,12 @@ public void testAutoCreateLeafQueueCreation() throws Exception { 3, 1); final CSQueue autoCreatedLeafQueue2 = cs.getQueue(TEST_GROUPUSER2); - validateCapacities((AutoCreatedLeafQueue) autoCreatedLeafQueue2, - 0.33332032f, - 0.03333203f, 1f, 0.6f); - validateCapacities((AutoCreatedLeafQueue) autoCreatedLeafQueue1, - 0.33332032f, - 0.03333203f, 1f, 0.6f); + validateCapacities((AutoCreatedLeafQueue) autoCreatedLeafQueue2, 0.0f, + 0.0f, 1f, 0.6f); + validateCapacities((AutoCreatedLeafQueue) autoCreatedLeafQueue1, 0.4f, + 0.04f, 1f, 0.6f); + validateCapacities((AutoCreatedLeafQueue) autoCreatedLeafQueue, 0.4f, + 0.04f, 1f, 0.6f); GuaranteedOrZeroCapacityOverTimePolicy autoCreatedQueueManagementPolicy = (GuaranteedOrZeroCapacityOverTimePolicy) ((ManagedParentQueue) parentQueue)