big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-6769. Make schedulables without demand less needy in FairSharePolicy#compare. (Yunfan Zhou via Yufei Gu)

Browse Source
This commit is contained in:
Yufei Gu 2017-07-13 23:10:10 -07:00
parent 228ddaa31d
commit 4a574e9a84
2 changed files with 27 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
View File

@ -59,6 +59,9 @@ public String getName() {
* Compare Schedulables via weighted fair sharing. In addition, Schedulables
* below their min share get priority over those whose min share is met.
*
* Schedulables without resource demand get lower priority than
* ones who have demands.
*
* Schedulables below their min share are compared by how far below it they
* are as a ratio. For example, if job A has 8 out of a min share of 10 tasks
* and job B has 50 out of a min share of 100, then job B is scheduled next,
@ -79,6 +82,16 @@ private static class FairShareComparator implements Comparator<Schedulable>,
@Override
public int compare(Schedulable s1, Schedulable s2) {
Resource demand1 = s1.getDemand();
Resource demand2 = s2.getDemand();
if (demand1.equals(Resources.none()) && Resources.greaterThan(
RESOURCE_CALCULATOR, null, demand2, Resources.none())) {
return 1;
} else if (demand2.equals(Resources.none()) && Resources.greaterThan(
RESOURCE_CALCULATOR, null, demand1, Resources.none())) {
return -1;
}
double minShareRatio1, minShareRatio2;
double useToWeightRatio1, useToWeightRatio2;
double weight1, weight2;
@ -86,9 +99,9 @@ public int compare(Schedulable s1, Schedulable s2) {
Resource resourceUsage1 = s1.getResourceUsage();
Resource resourceUsage2 = s2.getResourceUsage();
Resource minShare1 = Resources.min(RESOURCE_CALCULATOR, null,
s1.getMinShare(), s1.getDemand());
s1.getMinShare(), demand1);
Resource minShare2 = Resources.min(RESOURCE_CALCULATOR, null,
s2.getMinShare(), s2.getDemand());
s2.getMinShare(), demand2);
boolean s1Needy = Resources.lessThan(RESOURCE_CALCULATOR, null,
resourceUsage1, minShare1);
boolean s2Needy = Resources.lessThan(RESOURCE_CALCULATOR, null,

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java
View File

@ -123,6 +123,8 @@ private class FairShareComparatorTester {
private Resource minShare = Resource.newInstance(0, 1);
private Resource demand = Resource.newInstance(4, 1);
private Resource[] demandCollection = {
Resource.newInstance(0, 0), Resource.newInstance(4, 1) };
private String[] nameCollection = {"A", "B", "C"};
@ -160,21 +162,24 @@ private void generateAndTest(Stack<Schedulable> genSchedulable) {
for (int j = 0; j < startTimeColloection.length; j++) {
for (int k = 0; k < usageCollection.length; k++) {
for (int t = 0; t < weightsCollection.length; t++) {
genSchedulable.push(createSchedulable(i, j, k, t));
for (int m = 0; m < demandCollection.length; m++) {
genSchedulable.push(createSchedulable(m, i, j, k, t));
generateAndTest(genSchedulable);
genSchedulable.pop();
}
}
}
}
}
}
private Schedulable createSchedulable(
int nameIdx, int startTimeIdx, int usageIdx, int weightsIdx) {
return new MockSchedulable(minShare, demand, nameCollection[nameIdx],
startTimeColloection[startTimeIdx], usageCollection[usageIdx],
weightsCollection[weightsIdx]);
int demandId, int nameIdx, int startTimeIdx,
int usageIdx, int weightsIdx) {
return new MockSchedulable(minShare, demandCollection[demandId],
nameCollection[nameIdx], startTimeColloection[startTimeIdx],
usageCollection[usageIdx], weightsCollection[weightsIdx]);
}
private boolean checkTransitivity(