big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-3733. Fix DominantRC#compare() does not work as expected if cluster resource is empty. (Rohith Sharmaks via wangda)

Browse Source
This commit is contained in:
Wangda Tan 2015-06-04 10:22:57 -07:00
parent dbed757cba
commit ebd797c48f
3 changed files with 75 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -623,6 +623,9 @@ Release 2.7.1 - UNRELEASED
YARN-3585. NodeManager cannot exit on SHUTDOWN event triggered and NM
recovery is enabled (Rohith Sharmaks via jlowe)
YARN-3733. Fix DominantRC#compare() does not work as expected if
cluster resource is empty. (Rohith Sharmaks via wangda)
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
View File

@ -53,6 +53,21 @@ public int compare(Resource clusterResource, Resource lhs, Resource rhs) {
return 0;
}
if (isInvalidDivisor(clusterResource)) {
if ((lhs.getMemory() < rhs.getMemory() && lhs.getVirtualCores() > rhs
.getVirtualCores())
|| (lhs.getMemory() > rhs.getMemory() && lhs.getVirtualCores() < rhs
.getVirtualCores())) {
return 0;
} else if (lhs.getMemory() > rhs.getMemory()
|| lhs.getVirtualCores() > rhs.getVirtualCores()) {
return 1;
} else if (lhs.getMemory() < rhs.getMemory()
|| lhs.getVirtualCores() < rhs.getVirtualCores()) {
return -1;
}
}
float l = getResourceAsValue(clusterResource, lhs, true);
float r = getResourceAsValue(clusterResource, rhs, true);

58
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
View File

@ -130,6 +130,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfoList;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FairOrderingPolicy;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.After;
@ -1281,9 +1282,15 @@ public void testRecoverRequestAfterPreemption() throws Exception {
private MockRM setUpMove() {
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
return setUpMove(conf);
}
private MockRM setUpMove(Configuration config) {
CapacitySchedulerConfiguration conf =
new CapacitySchedulerConfiguration(config);
setupQueueConfiguration(conf);
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
ResourceScheduler.class);
ResourceScheduler.class);
MockRM rm = new MockRM(conf);
rm.start();
return rm;
@ -2952,6 +2959,55 @@ public void testDefaultNodeLabelExpressionQueueConfig() throws Exception {
Assert.assertEquals(queueInfoB.getDefaultNodeLabelExpression(), "y");
}
@Test(timeout = 30000)
public void testAMLimitUsage() throws Exception {
CapacitySchedulerConfiguration config =
new CapacitySchedulerConfiguration();
config.set(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
DefaultResourceCalculator.class.getName());
verifyAMLimitForLeafQueue(config);
config.set(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
DominantResourceCalculator.class.getName());
verifyAMLimitForLeafQueue(config);
}
private void verifyAMLimitForLeafQueue(CapacitySchedulerConfiguration config)
throws Exception {
MockRM rm = setUpMove(config);
String queueName = "a1";
String userName = "user_0";
ResourceScheduler scheduler = rm.getRMContext().getScheduler();
LeafQueue queueA =
(LeafQueue) ((CapacityScheduler) scheduler).getQueue(queueName);
Resource amResourceLimit = queueA.getAMResourceLimit();
Resource amResource =
Resource.newInstance(amResourceLimit.getMemory() + 1,
amResourceLimit.getVirtualCores() + 1);
rm.submitApp(amResource.getMemory(), "app-1", userName, null, queueName);
rm.submitApp(amResource.getMemory(), "app-1", userName, null, queueName);
// When AM limit is exceeded, 1 applications will be activated.Rest all
// applications will be in pending
Assert.assertEquals("PendingApplications should be 1", 1,
queueA.getNumPendingApplications());
Assert.assertEquals("Active applications should be 1", 1,
queueA.getNumActiveApplications());
Assert.assertEquals("User PendingApplications should be 1", 1, queueA
.getUser(userName).getPendingApplications());
Assert.assertEquals("User Active applications should be 1", 1, queueA
.getUser(userName).getActiveApplications());
rm.stop();
}
private void setMaxAllocMb(Configuration conf, int maxAllocMb) {
conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
maxAllocMb);