big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Addendum patch for YARN-5731

Browse Source
This commit is contained in:
Sunil G 2017-07-18 11:49:09 +05:30
parent ccaf036662
commit 0b7afc060c
3 changed files with 36 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
View File

@ -231,28 +231,13 @@ public void init(Configuration config, RMContext context,
.add(new ReservedContainerCandidatesSelector(this));
}
boolean additionalPreemptionBasedOnReservedResource = csConfig.getBoolean(
CapacitySchedulerConfiguration.ADDITIONAL_RESOURCE_BALANCE_BASED_ON_RESERVED_CONTAINERS,
CapacitySchedulerConfiguration.DEFAULT_ADDITIONAL_RESOURCE_BALANCE_BASED_ON_RESERVED_CONTAINERS);
// initialize candidates preemption selection policies
// When select candidates for reserved containers is enabled, exclude reserved
// resource in fifo policy (less aggressive). Otherwise include reserved
// resource.
//
// Why doing this? In YARN-4390, we added preemption-based-on-reserved-container
// Support. To reduce unnecessary preemption for large containers. We will
// not include reserved resources while calculating ideal-allocation in
// FifoCandidatesSelector.
//
// Changes in YARN-4390 will significantly reduce number of containers preempted
// When cluster has heterogeneous container requests. (Please check test
// report: https://issues.apache.org/jira/secure/attachment/12796197/YARN-4390-test-results.pdf
//
// However, on the other hand, in some corner cases, especially for
// fragmented cluster. It could lead to preemption cannot kick in in some
// cases. Please see YARN-5731.
//
// So to solve the problem, we will include reserved when surgical preemption
// for reserved container, which reverts behavior when YARN-4390 is disabled.
candidatesSelectionPolicies.add(new FifoCandidatesSelector(this,
!selectCandidatesForResevedContainers));
additionalPreemptionBasedOnReservedResource));
// Do we need to specially consider intra queue
boolean isIntraQueuePreemptionEnabled = csConfig.getBoolean(

27
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
View File

@ -1205,6 +1205,33 @@ public boolean getLazyPreemptionEnabled() {
public static final float DEFAULT_PREEMPTION_NATURAL_TERMINATION_FACTOR =
0.2f;
/**
* By default, reserved resource will be excluded while balancing capacities
* of queues.
*
* Why doing this? In YARN-4390, we added preemption-based-on-reserved-container
* Support. To reduce unnecessary preemption for large containers. We will
* not include reserved resources while calculating ideal-allocation in
* FifoCandidatesSelector.
*
* Changes in YARN-4390 will significantly reduce number of containers preempted
* When cluster has heterogeneous container requests. (Please check test
* report: https://issues.apache.org/jira/secure/attachment/12796197/YARN-4390-test-results.pdf
*
* However, on the other hand, in some corner cases, especially for
* fragmented cluster. It could lead to preemption cannot kick in in some
* cases. Please see YARN-5731.
*
* So to solve the problem, make this change to be configurable, and please
* note that it is an experimental option.
*/
public static final String
ADDITIONAL_RESOURCE_BALANCE_BASED_ON_RESERVED_CONTAINERS =
PREEMPTION_CONFIG_PREFIX
+ "additional_res_balance_based_on_reserved_containers";
public static final boolean
DEFAULT_ADDITIONAL_RESOURCE_BALANCE_BASED_ON_RESERVED_CONTAINERS = false;
/**
* When calculating which containers to be preempted, we will try to preempt
* containers for reserved containers first. By default is false.

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
View File

@ -813,9 +813,11 @@ public void testPriorityPreemptionFromHighestPriorityQueueAndOldestContainer()
@Test(timeout = 60000)
public void testPreemptionForFragmentatedCluster() throws Exception {
// Set additional_balance_queue_based_on_reserved_res to true to get
// additional preemptions.
conf.setBoolean(
CapacitySchedulerConfiguration.PREEMPTION_SELECT_CANDIDATES_FOR_RESERVED_CONTAINERS,
false);
CapacitySchedulerConfiguration.ADDITIONAL_RESOURCE_BALANCE_BASED_ON_RESERVED_CONTAINERS,
true);
/**
* Two queues, a/b, each of them are 50/50