big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10622. Fix preemption policy to exclude childless ParentQueues. Contributed by Andras Gyori

Browse Source
This commit is contained in:
Szilard Nemeth 2021-02-15 14:48:42 +01:00
parent 852aac34f2
commit bad6038a48
2 changed files with 48 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
View File

@ -17,6 +17,7 @@
*/ */
package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity; package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
import org.apache.commons.collections.CollectionUtils;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
@ -409,9 +410,11 @@ private void cleanupStaledPreemptionCandidates(long currentTime) {
} }
private Set<String> getLeafQueueNames(TempQueuePerPartition q) { private Set<String> getLeafQueueNames(TempQueuePerPartition q) {
// If its a ManagedParentQueue, it might not have any children // Also exclude ParentQueues, which might be without children
if ((q.children == null || q.children.isEmpty()) if (CollectionUtils.isEmpty(q.children)
&& !(q.parentQueue instanceof ManagedParentQueue)) { && !(q.parentQueue instanceof ManagedParentQueue)
&& (q.parentQueue == null
|| !q.parentQueue.isEligibleForAutoQueueCreation())) {
return ImmutableSet.of(q.queueName); return ImmutableSet.of(q.queueName);
} }

42
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
View File

@ -81,6 +81,7 @@
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType.MARK_CONTAINER_FOR_KILLABLE; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType.MARK_CONTAINER_FOR_KILLABLE;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType.MARK_CONTAINER_FOR_PREEMPTION; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType.MARK_CONTAINER_FOR_PREEMPTION;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail; import static org.junit.Assert.fail;
@ -1071,6 +1072,47 @@ public void testRefreshPreemptionProperties() throws Exception {
assertEquals(newObserveOnly, policy.isObserveOnly()); assertEquals(newObserveOnly, policy.isObserveOnly());
} }
@Test
public void testLeafQueueNameExtraction() throws Exception {
ProportionalCapacityPreemptionPolicy policy =
buildPolicy(Q_DATA_FOR_IGNORE);
ParentQueue root = (ParentQueue) mCS.getRootQueue();
root.addDynamicParentQueue("childlessFlexible");
List<CSQueue> queues = root.getChildQueues();
ArrayList<CSQueue> extendedQueues = new ArrayList<>();
LinkedList<ParentQueue> pqs = new LinkedList<>();
ParentQueue dynamicParent = mockParentQueue(
null, 0, pqs);
when(dynamicParent.getQueuePath()).thenReturn("root.dynamicParent");
when(dynamicParent.getQueueCapacities()).thenReturn(
new QueueCapacities(false));
QueueResourceQuotas dynamicParentQr = new QueueResourceQuotas();
dynamicParentQr.setEffectiveMaxResource(Resource.newInstance(1, 1));
dynamicParentQr.setEffectiveMinResource(Resources.createResource(1));
dynamicParentQr.setEffectiveMaxResource(RMNodeLabelsManager.NO_LABEL,
Resource.newInstance(1, 1));
dynamicParentQr.setEffectiveMinResource(RMNodeLabelsManager.NO_LABEL,
Resources.createResource(1));
when(dynamicParent.getQueueResourceQuotas()).thenReturn(dynamicParentQr);
when(dynamicParent.getEffectiveCapacity(RMNodeLabelsManager.NO_LABEL))
.thenReturn(Resources.createResource(1));
when(dynamicParent.getEffectiveMaxCapacity(RMNodeLabelsManager.NO_LABEL))
.thenReturn(Resource.newInstance(1, 1));
ResourceUsage resUsage = new ResourceUsage();
resUsage.setUsed(Resources.createResource(1024));
resUsage.setReserved(Resources.createResource(1024));
when(dynamicParent.getQueueResourceUsage()).thenReturn(resUsage);
when(dynamicParent.isEligibleForAutoQueueCreation()).thenReturn(true);
extendedQueues.add(dynamicParent);
extendedQueues.addAll(queues);
when(root.getChildQueues()).thenReturn(extendedQueues);
policy.editSchedule();
assertFalse("dynamicParent should not be a LeafQueue " +
"candidate", policy.getLeafQueueNames().contains("root.dynamicParent"));
}
static class IsPreemptionRequestFor static class IsPreemptionRequestFor
implements ArgumentMatcher<ContainerPreemptEvent> { implements ArgumentMatcher<ContainerPreemptEvent> {
private final ApplicationAttemptId appAttId; private final ApplicationAttemptId appAttId;