big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-8292: Fix the dominant resource preemption cannot happen when some of the resource vector becomes negative. Contributed by Wangda Tan.

Browse Source
This commit is contained in:
Eric E Payne 2018-05-25 16:06:09 +00:00
parent bddfe796f2
commit 8d5509c681
14 changed files with 312 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
View File

@ -135,14 +135,19 @@ public boolean fitsIn(Resource smaller, Resource bigger) {
return smaller.getMemorySize() <= bigger.getMemorySize();
}
@Override
public boolean isAnyMajorResourceZero(Resource resource) {
return resource.getMemorySize() == 0f;
}
@Override
public Resource normalizeDown(Resource r, Resource stepFactor) {
return Resources.createResource(
roundDown((r.getMemorySize()), stepFactor.getMemorySize()));
}
@Override
public boolean isAnyMajorResourceZeroOrNegative(Resource resource) {
return resource.getMemorySize() <= 0;
}
@Override
public boolean isAnyMajorResourceAboveZero(Resource resource) {
return resource.getMemorySize() > 0;
}
}

39
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
View File

@ -576,19 +576,6 @@ public boolean fitsIn(Resource smaller, Resource bigger) {
return true;
}
@Override
public boolean isAnyMajorResourceZero(Resource resource) {
int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
for (int i = 0; i < maxLength; i++) {
ResourceInformation resourceInformation = resource
.getResourceInformation(i);
if (resourceInformation.getValue() == 0L) {
return true;
}
}
return false;
}
@Override
public Resource normalizeDown(Resource r, Resource stepFactor) {
Resource ret = Resource.newInstance(r);
@ -613,4 +600,30 @@ public Resource normalizeDown(Resource r, Resource stepFactor) {
}
return ret;
}
@Override
public boolean isAnyMajorResourceZeroOrNegative(Resource resource) {
int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
for (int i = 0; i < maxLength; i++) {
ResourceInformation resourceInformation = resource.getResourceInformation(
i);
if (resourceInformation.getValue() <= 0L) {
return true;
}
}
return false;
}
@Override
public boolean isAnyMajorResourceAboveZero(Resource resource) {
int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
for (int i = 0; i < maxLength; i++) {
ResourceInformation resourceInformation = resource.getResourceInformation(
i);
if (resourceInformation.getValue() > 0) {
return true;
}
}
return false;
}
}

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
View File

@ -239,12 +239,12 @@ public abstract float divide(
/**
* Check if resource has any major resource types (which are all NodeManagers
* included) a zero value.
* included) a zero value or negative value.
*
* @param resource resource
* @return returns true if any resource is zero.
*/
public abstract boolean isAnyMajorResourceZero(Resource resource);
public abstract boolean isAnyMajorResourceZeroOrNegative(Resource resource);
/**
* Get resource <code>r</code>and normalize down using step-factor
@ -257,4 +257,13 @@ public abstract float divide(
* @return resulting normalized resource
*/
public abstract Resource normalizeDown(Resource r, Resource stepFactor);
/**
* Check if resource has any major resource types (which are all NodeManagers
* included) has a >0 value.
*
* @param resource resource
* @return returns true if any resource is >0
*/
public abstract boolean isAnyMajorResourceAboveZero(Resource resource);
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
View File

@ -547,11 +547,6 @@ public static Resource componentwiseMax(Resource lhs, Resource rhs) {
return ret;
}
public static boolean isAnyMajorResourceZero(ResourceCalculator rc,
Resource resource) {
return rc.isAnyMajorResourceZero(resource);
}
public static Resource normalizeDown(ResourceCalculator calculator,
Resource resource, Resource factor) {
return calculator.normalizeDown(resource, factor);

58
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java
View File

@ -18,12 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.Iterator;
import java.util.PriorityQueue;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy.PriorityUtilizationQueueOrderingPolicy;
@ -32,6 +26,12 @@
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.Iterator;
import java.util.PriorityQueue;
/**
* Calculate how much resources need to be preempted for each queue,
* will be used by {@link PreemptionCandidatesSelector}.
@ -40,7 +40,8 @@ public class AbstractPreemptableResourceCalculator {
protected final CapacitySchedulerPreemptionContext context;
protected final ResourceCalculator rc;
private boolean isReservedPreemptionCandidatesSelector;
protected boolean isReservedPreemptionCandidatesSelector;
private Resource stepFactor;
static class TQComparator implements Comparator<TempQueuePerPartition> {
private ResourceCalculator rc;
@ -90,6 +91,11 @@ public AbstractPreemptableResourceCalculator(
rc = preemptionContext.getResourceCalculator();
this.isReservedPreemptionCandidatesSelector =
isReservedPreemptionCandidatesSelector;
stepFactor = Resource.newInstance(0, 0);
for (ResourceInformation ri : stepFactor.getResources()) {
ri.setValue(1);
}
}
/**
@ -122,23 +128,24 @@ protected void computeFixpointAllocation(Resource totGuarant,
TQComparator tqComparator = new TQComparator(rc, totGuarant);
PriorityQueue<TempQueuePerPartition> orderedByNeed = new PriorityQueue<>(10,
tqComparator);
for (Iterator<TempQueuePerPartition> i = qAlloc.iterator(); i.hasNext();) {
for (Iterator<TempQueuePerPartition> i = qAlloc.iterator(); i.hasNext(); ) {
TempQueuePerPartition q = i.next();
Resource used = q.getUsed();
Resource initIdealAssigned;
if (Resources.greaterThan(rc, totGuarant, used, q.getGuaranteed())) {
initIdealAssigned =
Resources.add(q.getGuaranteed(), q.untouchableExtra);
} else {
initIdealAssigned = Resources.add(
Resources.componentwiseMin(q.getGuaranteed(), q.getUsed()),
q.untouchableExtra);
} else{
initIdealAssigned = Resources.clone(used);
}
// perform initial assignment
initIdealAssignment(totGuarant, q, initIdealAssigned);
Resources.subtractFrom(unassigned, q.idealAssigned);
// If idealAssigned < (allocated + used + pending), q needs more
// resources, so
// add it to the list of underserved queues, ordered by need.
@ -152,7 +159,6 @@ protected void computeFixpointAllocation(Resource totGuarant,
// left
while (!orderedByNeed.isEmpty() && Resources.greaterThan(rc, totGuarant,
unassigned, Resources.none())) {
Resource wQassigned = Resource.newInstance(0, 0);
// we compute normalizedGuarantees capacity based on currently active
// queues
resetCapacity(unassigned, orderedByNeed, ignoreGuarantee);
@ -166,11 +172,26 @@ protected void computeFixpointAllocation(Resource totGuarant,
Collection<TempQueuePerPartition> underserved = getMostUnderservedQueues(
orderedByNeed, tqComparator);
// This value will be used in every round to calculate ideal allocation.
// So make a copy to avoid it changed during calculation.
Resource dupUnassignedForTheRound = Resources.clone(unassigned);
for (Iterator<TempQueuePerPartition> i = underserved.iterator(); i
.hasNext();) {
if (!rc.isAnyMajorResourceAboveZero(unassigned)) {
break;
}
TempQueuePerPartition sub = i.next();
Resource wQavail = Resources.multiplyAndNormalizeUp(rc, unassigned,
sub.normalizedGuarantee, Resource.newInstance(1, 1));
// How much resource we offer to the queue (to increase its ideal_alloc
Resource wQavail = Resources.multiplyAndNormalizeUp(rc,
dupUnassignedForTheRound,
sub.normalizedGuarantee, this.stepFactor);
// Make sure it is not beyond unassigned
wQavail = Resources.componentwiseMin(wQavail, unassigned);
Resource wQidle = sub.offer(wQavail, rc, totGuarant,
isReservedPreemptionCandidatesSelector);
Resource wQdone = Resources.subtract(wQavail, wQidle);
@ -180,9 +201,12 @@ protected void computeFixpointAllocation(Resource totGuarant,
// queue, recalculating its order based on need.
orderedByNeed.add(sub);
}
Resources.addTo(wQassigned, wQdone);
Resources.subtractFrom(unassigned, wQdone);
// Make sure unassigned is always larger than 0
unassigned = Resources.componentwiseMax(unassigned, Resources.none());
}
Resources.subtractFrom(unassigned, wQassigned);
}
// Sometimes its possible that, all queues are properly served. So intra

61
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/CapacitySchedulerPreemptionUtils.java
View File

@ -21,6 +21,7 @@
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
@ -132,6 +133,16 @@ private static void deductPreemptableResourcePerApp(
* map to hold preempted containers
* @param totalPreemptionAllowed
* total preemption allowed per round
* @param conservativeDRF
* should we do conservativeDRF preemption or not.
* When true:
* stop preempt container when any major resource type <= 0 for to-
* preempt.
* This is default preemption behavior of intra-queue preemption
* When false:
* stop preempt container when: all major resource type <= 0 for
* to-preempt.
* This is default preemption behavior of inter-queue preemption
* @return should we preempt rmContainer. If we should, deduct from
* <code>resourceToObtainByPartition</code>
*/
@ -140,7 +151,7 @@ public static boolean tryPreemptContainerAndDeductResToObtain(
Map<String, Resource> resourceToObtainByPartitions,
RMContainer rmContainer, Resource clusterResource,
Map<ApplicationAttemptId, Set<RMContainer>> preemptMap,
Resource totalPreemptionAllowed) {
Resource totalPreemptionAllowed, boolean conservativeDRF) {
ApplicationAttemptId attemptId = rmContainer.getApplicationAttemptId();
// We will not account resource of a container twice or more
@ -152,13 +163,49 @@ public static boolean tryPreemptContainerAndDeductResToObtain(
rmContainer.getAllocatedNode());
Resource toObtainByPartition = resourceToObtainByPartitions
.get(nodePartition);
if (null == toObtainByPartition) {
return false;
}
// If a toObtain resource type == 0, set it to -1 to avoid 0 resource
// type affect following doPreemption check: isAnyMajorResourceZero
for (ResourceInformation ri : toObtainByPartition.getResources()) {
if (ri.getValue() == 0) {
ri.setValue(-1);
}
}
if (rc.isAnyMajorResourceAboveZero(toObtainByPartition) && Resources.fitsIn(
rc, rmContainer.getAllocatedResource(), totalPreemptionAllowed)) {
boolean doPreempt;
// How much resource left after preemption happen.
Resource toObtainAfterPreemption = Resources.subtract(toObtainByPartition,
rmContainer.getAllocatedResource());
if (conservativeDRF) {
doPreempt = !rc.isAnyMajorResourceZeroOrNegative(toObtainByPartition);
} else {
// When we want to do more aggressive preemption, we will do preemption
// only if:
// - The preempt of the container makes positive contribution to the
// to-obtain resource. Positive contribution means any positive
// resource type decreases.
//
// This is example of positive contribution:
// * before: <30, 10, 5>, after <20, 10, -10>
// But this not positive contribution:
// * before: <30, 10, 0>, after <30, 10, -15>
doPreempt = Resources.lessThan(rc, clusterResource,
Resources
.componentwiseMax(toObtainAfterPreemption, Resources.none()),
Resources.componentwiseMax(toObtainByPartition, Resources.none()));
}
if (!doPreempt) {
return false;
}
if (null != toObtainByPartition
&& Resources.greaterThan(rc, clusterResource, toObtainByPartition,
Resources.none())
&& Resources.fitsIn(rc, rmContainer.getAllocatedResource(),
totalPreemptionAllowed)
&& !Resources.isAnyMajorResourceZero(rc, toObtainByPartition)) {
Resources.subtractFrom(toObtainByPartition,
rmContainer.getAllocatedResource());
Resources.subtractFrom(totalPreemptionAllowed,

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java
View File

@ -111,7 +111,7 @@ public Map<ApplicationAttemptId, Set<RMContainer>> selectCandidates(
.tryPreemptContainerAndDeductResToObtain(rc,
preemptionContext, resToObtainByPartition, c,
clusterResource, selectedCandidates,
totalPreemptionAllowed);
totalPreemptionAllowed, false);
if (!preempted) {
continue;
}
@ -187,7 +187,7 @@ private void preemptAMContainers(Resource clusterResource,
boolean preempted = CapacitySchedulerPreemptionUtils
.tryPreemptContainerAndDeductResToObtain(rc, preemptionContext,
resToObtainByPartition, c, clusterResource, preemptMap,
totalPreemptionAllowed);
totalPreemptionAllowed, false);
if (preempted) {
Resources.subtractFrom(skippedAMSize, c.getAllocatedResource());
}
@ -221,7 +221,7 @@ private void preemptFrom(FiCaSchedulerApp app,
// Try to preempt this container
CapacitySchedulerPreemptionUtils.tryPreemptContainerAndDeductResToObtain(
rc, preemptionContext, resToObtainByPartition, c, clusterResource,
selectedContainers, totalPreemptionAllowed);
selectedContainers, totalPreemptionAllowed, false);
if (!preemptionContext.isObserveOnly()) {
preemptionContext.getRMContext().getDispatcher().getEventHandler()
@ -264,7 +264,7 @@ private void preemptFrom(FiCaSchedulerApp app,
// Try to preempt this container
CapacitySchedulerPreemptionUtils.tryPreemptContainerAndDeductResToObtain(
rc, preemptionContext, resToObtainByPartition, c, clusterResource,
selectedContainers, totalPreemptionAllowed);
selectedContainers, totalPreemptionAllowed, false);
}
}
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java
View File

@ -278,8 +278,8 @@ private TreeSet<TempAppPerPartition> calculateIdealAssignedResourcePerApp(
// Once unallocated resource is 0, we can stop assigning ideal per app.
if (Resources.lessThanOrEqual(rc, clusterResource,
queueReassignableResource, Resources.none())
|| Resources.isAnyMajorResourceZero(rc, queueReassignableResource)) {
queueReassignableResource, Resources.none()) || rc
.isAnyMajorResourceZeroOrNegative(queueReassignableResource)) {
continue;
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/IntraQueueCandidatesSelector.java
View File

@ -230,7 +230,7 @@ private void preemptFromLeastStarvedApp(LeafQueue leafQueue,
boolean ret = CapacitySchedulerPreemptionUtils
.tryPreemptContainerAndDeductResToObtain(rc, preemptionContext,
resToObtainByPartition, c, clusterResource, selectedCandidates,
totalPreemptedResourceAllowed);
totalPreemptedResourceAllowed, true);
// Subtract from respective user's resource usage once a container is
// selected for preemption.

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/PreemptableResourceCalculator.java
View File

@ -41,8 +41,6 @@ public class PreemptableResourceCalculator
private static final Log LOG =
LogFactory.getLog(PreemptableResourceCalculator.class);
private boolean isReservedPreemptionCandidatesSelector;
/**
* PreemptableResourceCalculator constructor
*
@ -95,8 +93,8 @@ protected void computeIdealResourceDistribution(ResourceCalculator rc,
}
// first compute the allocation as a fixpoint based on guaranteed capacity
computeFixpointAllocation(tot_guarant, nonZeroGuarQueues, unassigned,
false);
computeFixpointAllocation(tot_guarant, new HashSet<>(nonZeroGuarQueues),
unassigned, false);
// if any capacity is left unassigned, distributed among zero-guarantee
// queues uniformly (i.e., not based on guaranteed capacity, as this is zero)

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
View File

@ -151,7 +151,7 @@ Resource offer(Resource avail, ResourceCalculator rc,
// # This is for leaf queue only.
// max(guaranteed, used) - assigned}
// remain = avail - accepted
Resource accepted = Resources.min(rc, clusterResource,
Resource accepted = Resources.componentwiseMin(
absMaxCapIdealAssignedDelta,
Resources.min(rc, clusterResource, avail, Resources
/*
@ -186,6 +186,12 @@ Resource offer(Resource avail, ResourceCalculator rc,
accepted = acceptedByLocality(rc, accepted);
// accept should never be < 0
accepted = Resources.componentwiseMax(accepted, Resources.none());
// or more than offered
accepted = Resources.componentwiseMin(accepted, avail);
Resource remain = Resources.subtract(avail, accepted);
Resources.addTo(idealAssigned, accepted);
return remain;

30
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
View File

@ -29,6 +29,7 @@
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
@ -59,6 +60,7 @@
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.mockito.ArgumentMatcher;
@ -104,10 +106,32 @@ public class ProportionalCapacityPreemptionPolicyMockFramework {
EventHandler<Event> mDisp = null;
ProportionalCapacityPreemptionPolicy policy = null;
Resource clusterResource = null;
// Initialize resource map
Map<String, ResourceInformation> riMap = new HashMap<>();
private void resetResourceInformationMap() {
// Initialize mandatory resources
ResourceInformation memory = ResourceInformation.newInstance(
ResourceInformation.MEMORY_MB.getName(),
ResourceInformation.MEMORY_MB.getUnits(),
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
ResourceInformation vcores = ResourceInformation.newInstance(
ResourceInformation.VCORES.getName(),
ResourceInformation.VCORES.getUnits(),
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
riMap.put(ResourceInformation.MEMORY_URI, memory);
riMap.put(ResourceInformation.VCORES_URI, vcores);
ResourceUtils.initializeResourcesFromResourceInformationMap(riMap);
}
@SuppressWarnings("unchecked")
@Before
public void setup() {
resetResourceInformationMap();
org.apache.log4j.Logger.getRootLogger().setLevel(
org.apache.log4j.Level.DEBUG);
@ -142,6 +166,12 @@ public void setup() {
partitionToResource = new HashMap<>();
nodeIdToSchedulerNodes = new HashMap<>();
nameToCSQueues = new HashMap<>();
clusterResource = Resource.newInstance(0, 0);
}
@After
public void cleanup() {
resetResourceInformationMap();
}
public void buildEnv(String labelsConfig, String nodesConfig,

103
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestPreemptionForQueueWithPriorities.java
View File

@ -20,44 +20,25 @@
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import static org.mockito.Matchers.argThat;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
public class TestPreemptionForQueueWithPriorities
extends ProportionalCapacityPreemptionPolicyMockFramework {
// Initialize resource map
private Map<String, ResourceInformation> riMap = new HashMap<>();
@Before
public void setup() {
// Initialize mandatory resources
ResourceInformation memory = ResourceInformation.newInstance(
ResourceInformation.MEMORY_MB.getName(),
ResourceInformation.MEMORY_MB.getUnits(),
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
ResourceInformation vcores = ResourceInformation.newInstance(
ResourceInformation.VCORES.getName(),
ResourceInformation.VCORES.getUnits(),
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
riMap.put(ResourceInformation.MEMORY_URI, memory);
riMap.put(ResourceInformation.VCORES_URI, vcores);
ResourceUtils.initializeResourcesFromResourceInformationMap(riMap);
rc = new DefaultResourceCalculator();
super.setup();
policy = new ProportionalCapacityPreemptionPolicy(rmContext, cs, mClock);
}
@ -340,8 +321,8 @@ public void testPriorityPreemptionForHierarchicalOfQueues()
* - a2 (capacity=60), p=1
* - b (capacity=30), p=1
* - b1 (capacity=50), p=1
* - b1 (capacity=50), p=2
* - c (capacity=40), p=2
* - b2 (capacity=50), p=2
* - c (capacity=40), p=1
* </pre>
*/
String labelsConfig = "=100,true"; // default partition
@ -349,11 +330,11 @@ public void testPriorityPreemptionForHierarchicalOfQueues()
String queuesConfig =
// guaranteed,max,used,pending
"root(=[100 100 100 100]);" + //root
"-a(=[30 100 40 50]){priority=1};" + // a
"-a(=[29 100 40 50]){priority=1};" + // a
"--a1(=[12 100 20 50]){priority=1};" + // a1
"--a2(=[18 100 20 50]){priority=1};" + // a2
"-b(=[30 100 59 50]){priority=1};" + // b
"--b1(=[15 100 30 50]){priority=1};" + // b1
"--a2(=[17 100 20 50]){priority=1};" + // a2
"-b(=[31 100 59 50]){priority=1};" + // b
"--b1(=[16 100 30 50]){priority=1};" + // b1
"--b2(=[15 100 29 50]){priority=2};" + // b2
"-c(=[40 100 1 30]){priority=1}"; // c
String appsConfig =
@ -362,7 +343,7 @@ public void testPriorityPreemptionForHierarchicalOfQueues()
"a2\t(1,1,n1,,20,false);" + // app2 in a2
"b1\t(1,1,n1,,30,false);" + // app3 in b1
"b2\t(1,1,n1,,29,false);" + // app4 in b2
"c\t(1,1,n1,,29,false)"; // app5 in c
"c\t(1,1,n1,,1,false)"; // app5 in c
buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig);
@ -370,16 +351,16 @@ public void testPriorityPreemptionForHierarchicalOfQueues()
// Preemption should first divide capacities between a / b, and b2 should
// get less preemption than b1 (because b2 has higher priority)
verify(mDisp, times(5)).handle(argThat(
verify(mDisp, times(6)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(1))));
verify(mDisp, never()).handle(argThat(
verify(mDisp, times(1)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(2))));
verify(mDisp, times(15)).handle(argThat(
verify(mDisp, times(13)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(3))));
verify(mDisp, times(9)).handle(argThat(
verify(mDisp, times(10)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(4))));
}
@ -426,7 +407,7 @@ public void testPriorityPreemptionWithMandatoryResourceForHierarchicalOfQueues()
// Preemption should first divide capacities between a / b, and b1 should
// get less preemption than b2 (because b1 has higher priority)
verify(mDisp, never()).handle(argThat(
verify(mDisp, times(3)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(1))));
verify(mDisp, never()).handle(argThat(
@ -505,4 +486,56 @@ public void testPriorityPreemptionWithMultipleResource()
getAppAttemptId(3))));
}
@Test
public void test3ResourceTypesInterQueuePreemption() throws IOException {
rc = new DominantResourceCalculator();
when(cs.getResourceCalculator()).thenReturn(rc);
// Initialize resource map
String RESOURCE_1 = "res1";
riMap.put(RESOURCE_1, ResourceInformation.newInstance(RESOURCE_1, "", 0,
ResourceTypes.COUNTABLE, 0, Integer.MAX_VALUE));
ResourceUtils.initializeResourcesFromResourceInformationMap(riMap);
/**
* Queue structure is:
*
* <pre>
* root
* / \ \
* a b c
* </pre>
* A / B / C have 33.3 / 33.3 / 33.4 resources
* Total cluster resource have mem=30, cpu=18, GPU=6
* A uses mem=6, cpu=3, GPU=3
* B uses mem=6, cpu=3, GPU=3
* C is asking mem=1,cpu=1,GPU=1
*
* We expect it can preempt from one of the jobs
*/
String labelsConfig =
"=30:18:6,true;";
String nodesConfig =
"n1= res=30:18:6;"; // n1 is default partition
String queuesConfig =
// guaranteed,max,used,pending
"root(=[30:18:6 30:18:6 12:12:6 1:1:1]){priority=1};" + //root
"-a(=[10:6:2 10:6:2 6:6:3 0:0:0]){priority=1};" + // a
"-b(=[10:6:2 10:6:2 6:6:3 0:0:0]){priority=1};" + // b
"-c(=[10:6:2 10:6:2 0:0:0 1:1:1]){priority=2}"; // c
String appsConfig=
//queueName\t(priority,resource,host,expression,#repeat,reserved)
"a\t" // app1 in a1
+ "(1,2:2:1,n1,,3,false);" +
"b\t" // app2 in b2
+ "(1,2:2:1,n1,,3,false)";
buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig);
policy.editSchedule();
verify(mDisp, times(1)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(1))));
}
}

60
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyInterQueueWithDRF.java
View File

@ -18,11 +18,16 @@
package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.junit.Before;
import org.junit.Test;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import java.io.IOException;
import static org.mockito.Matchers.argThat;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
@ -41,8 +46,7 @@ public void setup() {
}
@Test
public void testInterQueuePreemptionWithMultipleResource()
throws Exception {
public void testInterQueuePreemptionWithMultipleResource() throws Exception {
/**
* Queue structure is:
*
@ -121,4 +125,52 @@ public void testInterQueuePreemptionWithNaturalTerminationFactor()
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(1))));
}
}
@Test
public void test3ResourceTypesInterQueuePreemption() throws IOException {
// Initialize resource map
String RESOURCE_1 = "res1";
riMap.put(RESOURCE_1, ResourceInformation
.newInstance(RESOURCE_1, "", 0, ResourceTypes.COUNTABLE, 0,
Integer.MAX_VALUE));
ResourceUtils.initializeResourcesFromResourceInformationMap(riMap);
/*
* root
* / \ \
* a b c
*
* A / B / C have 33.3 / 33.3 / 33.4 resources
* Total cluster resource have mem=30, cpu=18, GPU=6
* A uses mem=6, cpu=3, GPU=3
* B uses mem=6, cpu=3, GPU=3
* C is asking mem=1,cpu=1,GPU=1
*
* We expect it can preempt from one of the jobs
*/
String labelsConfig = "=30:18:6,true;";
String nodesConfig = "n1= res=30:18:6;"; // n1 is default partition
String queuesConfig =
// guaranteed,max,used,pending
"root(=[30:18:6 30:18:6 12:12:6 1:1:1]);" + //root
"-a(=[10:7:2 10:6:3 6:6:3 0:0:0]);" + // a
"-b(=[10:6:2 10:6:3 6:6:3 0:0:0]);" + // b
"-c(=[10:5:2 10:6:2 0:0:0 1:1:1])"; // c
String appsConfig =
//queueName\t(priority,resource,host,expression,#repeat,reserved)
"a\t" // app1 in a1
+ "(1,2:2:1,n1,,3,false);" + "b\t" // app2 in b2
+ "(1,2:2:1,n1,,3,false)";
buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig);
policy.editSchedule();
verify(mDisp, times(0)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(1))));
verify(mDisp, times(1)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(2))));
}
}