big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-7789. Should fail RM if 3rd resource type is configured but RM uses DefaultResourceCalculator. (Zian Chen via wangda)

Browse Source 
Change-Id: I1f6a451fe16758def6f47c046a9b8a67ad7255af
This commit is contained in:
Wangda Tan 2018-02-14 23:11:10 +08:00
parent 60971b8195
commit 042ef2fa7b
2 changed files with 45 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
View File

@ -338,6 +338,14 @@ void initScheduler(Configuration configuration) throws
this.minimumAllocation = super.getMinimumAllocation();
initMaximumResourceCapability(super.getMaximumAllocation());
this.calculator = this.conf.getResourceCalculator();
if (this.calculator instanceof DefaultResourceCalculator
&& ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
throw new YarnRuntimeException("RM uses DefaultResourceCalculator which"
+ " used only memory as resource-type but invalid resource-types"
+ " specified " + ResourceUtils.getResourceTypes() + ". Use"
+ " DomainantResourceCalculator instead to make effective use of"
+ " these resource-types");
}
this.usePortForNodeName = this.conf.getUsePortForNodeName();
this.applications = new ConcurrentHashMap<>();
this.labelManager = rmContext.getNodeLabelManager();

37
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
View File

@ -26,6 +26,7 @@
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
@ -38,6 +39,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.junit.Assert;
@ -286,4 +288,39 @@ public void testMaximumAllocationRefreshWithMultipleResourceTypes() throws Excep
rm.close();
}
@Test
public void testDefaultResourceCalculatorWithThirdResourceTypes() throws Exception {
CapacitySchedulerConfiguration csconf =
new CapacitySchedulerConfiguration();
csconf.setResourceComparator(DefaultResourceCalculator.class);
YarnConfiguration conf = new YarnConfiguration(csconf);
String[] res1 = {"resource1", "M"};
String[] res2 = {"resource2", "G"};
String[] res3 = {"resource3", "H"};
String[][] test = {res1, res2, res3};
String resSt = "";
for (String[] resources : test) {
resSt += (resources[0] + ",");
}
resSt = resSt.substring(0, resSt.length() - 1);
conf.set(YarnConfiguration.RESOURCE_TYPES, resSt);
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
ResourceScheduler.class);
boolean exception = false;
try {
MockRM rm = new MockRM(conf);
} catch (YarnRuntimeException e) {
exception = true;
}
Assert.assertTrue("Should have exception in CS", exception);
}
}