big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10587. Fix AutoCreateLeafQueueCreation cap related caculation when in absolute mode. (Qi Zhu via wangda)

Browse Source 
Change-Id: I8889c393a9f08b633c72a2e873c6f45ca12cc82b
This commit is contained in:
Wangda Tan 2021-01-21 09:31:14 -08:00
parent b7384a8d02
commit 4f008153ef
3 changed files with 19 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java
View File

@ -19,6 +19,7 @@
.queuemanagement;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
.QueueManagementDynamicEditPolicy;
@ -624,7 +625,7 @@ private Map<String, QueueCapacities> deactivateLeafQueuesIfInActive(
QueueCapacities capacities = leafQueueEntitlements.get(
leafQueue.getQueuePath());
updateToZeroCapacity(capacities, nodeLabel);
updateToZeroCapacity(capacities, nodeLabel, (LeafQueue)childQueue);
deactivatedQueues.put(leafQueue.getQueuePath(),
leafQueueTemplateCapacities);
}
@ -822,7 +823,7 @@ public AutoCreatedLeafQueueConfig getInitialLeafQueueConfiguration(
updateCapacityFromTemplate(capacities, nodeLabel);
activate(leafQueue, nodeLabel);
} else{
updateToZeroCapacity(capacities, nodeLabel);
updateToZeroCapacity(capacities, nodeLabel, leafQueue);
}
}
@ -834,10 +835,12 @@ public AutoCreatedLeafQueueConfig getInitialLeafQueueConfiguration(
}
private void updateToZeroCapacity(QueueCapacities capacities,
String nodeLabel) {
String nodeLabel, LeafQueue leafQueue) {
capacities.setCapacity(nodeLabel, 0.0f);
capacities.setMaximumCapacity(nodeLabel,
leafQueueTemplateCapacities.getMaximumCapacity(nodeLabel));
leafQueue.getQueueResourceQuotas().
setConfiguredMinResource(nodeLabel, Resource.newInstance(0, 0));
}
private void updateCapacityFromTemplate(QueueCapacities capacities,

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
View File

@ -231,8 +231,10 @@ public void testSimpleMinMaxResourceConfigurartionPerQueue()
d1.queueResourceQuotas.getEffectiveMaxResource());
/**
* After adding child queue d2, adjustment happens and both d1 and d2 shares
* resource of Managed Parent Queue
* After adding child queue d2, d1 + d2 > resource
* of Managed Parent queue, d2 will change to 0.
* d1 will occupy all entire resource
* of Managed Parent queue.
*/
AutoCreatedLeafQueue d2 = new AutoCreatedLeafQueue(cs, "d2", parentQueue);
cs.addQueue(d2);
@ -240,9 +242,9 @@ public void testSimpleMinMaxResourceConfigurartionPerQueue()
cs.getRootQueue().updateClusterResource(cs.getClusterResource(),
new ResourceLimits(cs.getClusterResource()));
Assert.assertEquals(QUEUE_D_TEMPL_MINRES,
Assert.assertEquals(Resource.newInstance(0, 0),
d2.queueResourceQuotas.getConfiguredMinResource());
Assert.assertEquals(Resource.newInstance(12800, 2),
Assert.assertEquals(Resource.newInstance(0, 0),
d2.queueResourceQuotas.getEffectiveMinResource());
Assert.assertEquals(QUEUE_D_TEMPL_MAXRES,
d2.queueResourceQuotas.getConfiguredMaxResource());
@ -251,7 +253,7 @@ public void testSimpleMinMaxResourceConfigurartionPerQueue()
Assert.assertEquals(QUEUE_D_TEMPL_MINRES,
d1.queueResourceQuotas.getConfiguredMinResource());
Assert.assertEquals(Resource.newInstance(12800, 2),
Assert.assertEquals(QUEUE_D_TEMPL_MINRES,
d1.queueResourceQuotas.getEffectiveMinResource());
Assert.assertEquals(QUEUE_D_TEMPL_MAXRES,
d1.queueResourceQuotas.getConfiguredMaxResource());

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java
View File

@ -235,12 +235,12 @@ public void testAutoCreateLeafQueueCreation() throws Exception {
3, 1);
final CSQueue autoCreatedLeafQueue2 = cs.getQueue(TEST_GROUPUSER2);
validateCapacities((AutoCreatedLeafQueue) autoCreatedLeafQueue2,
0.33332032f,
0.03333203f, 1f, 0.6f);
validateCapacities((AutoCreatedLeafQueue) autoCreatedLeafQueue1,
0.33332032f,
0.03333203f, 1f, 0.6f);
validateCapacities((AutoCreatedLeafQueue) autoCreatedLeafQueue2, 0.0f,
0.0f, 1f, 0.6f);
validateCapacities((AutoCreatedLeafQueue) autoCreatedLeafQueue1, 0.4f,
0.04f, 1f, 0.6f);
validateCapacities((AutoCreatedLeafQueue) autoCreatedLeafQueue, 0.4f,
0.04f, 1f, 0.6f);
GuaranteedOrZeroCapacityOverTimePolicy autoCreatedQueueManagementPolicy =
(GuaranteedOrZeroCapacityOverTimePolicy) ((ManagedParentQueue) parentQueue)