big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10157. FS-CS converter: initPropertyActions() is not called without rules file. Contributed by Peter Bacsko

Browse Source
This commit is contained in:
Szilard Nemeth 2020-02-24 21:45:49 +01:00
parent 929004074f
commit 34cf63c7d0
3 changed files with 40 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java
View File

@ -166,11 +166,13 @@ private Resource getClusterResource(
private void loadConversionRules(String rulesFile) throws IOException { private void loadConversionRules(String rulesFile) throws IOException {
if (rulesFile != null) { if (rulesFile != null) {
LOG.info("Reading conversion rules file from: " + rulesFile); LOG.info("Reading conversion rules file from: " + rulesFile);
this.ruleHandler.loadRulesFromFile(rulesFile); ruleHandler.loadRulesFromFile(rulesFile);
} else { } else {
LOG.info("Conversion rules file is not defined, " + LOG.info("Conversion rules file is not defined, " +
"using default conversion config!"); "using default conversion config!");
} }
ruleHandler.initPropertyActions();
} }
private Configuration getInputYarnSiteConfig( private Configuration getInputYarnSiteConfig(

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigRuleHandler.java
View File

@ -95,7 +95,6 @@ void loadRulesFromFile(String ruleFile) throws IOException {
properties.load(is); properties.load(is);
} }
actions = new HashMap<>(); actions = new HashMap<>();
initPropertyActions();
} }
public FSConfigToCSConfigRuleHandler(ConversionOptions conversionOptions) { public FSConfigToCSConfigRuleHandler(ConversionOptions conversionOptions) {
@ -113,7 +112,7 @@ public FSConfigToCSConfigRuleHandler(ConversionOptions conversionOptions) {
initPropertyActions(); initPropertyActions();
} }
private void initPropertyActions() { public void initPropertyActions() {
setActionForProperty(MAX_CAPACITY_PERCENTAGE); setActionForProperty(MAX_CAPACITY_PERCENTAGE);
setActionForProperty(MAX_CHILD_CAPACITY); setActionForProperty(MAX_CHILD_CAPACITY);
setActionForProperty(USER_MAX_RUNNING_APPS); setActionForProperty(USER_MAX_RUNNING_APPS);

36
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java
View File

@ -26,6 +26,7 @@
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.USER_MAX_APPS_DEFAULT; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.USER_MAX_APPS_DEFAULT;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.USER_MAX_RUNNING_APPS; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.USER_MAX_RUNNING_APPS;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.RuleAction.ABORT; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.RuleAction.ABORT;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.RuleAction.WARNING;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull; import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
@ -372,6 +373,41 @@ public void testConvertFSConfigurationRulesFile() throws Exception {
ABORT, actions.get(QUEUE_AUTO_CREATE)); ABORT, actions.get(QUEUE_AUTO_CREATE));
} }
@Test
public void testConvertFSConfigurationWithoutRulesFile() throws Exception {
ruleHandler = new FSConfigToCSConfigRuleHandler(
createDefaultConversionOptions());
createConverter();
FSConfigToCSConfigConverterParams params =
createDefaultParamsBuilder()
.withClusterResource(CLUSTER_RESOURCE_STRING)
.build();
converter.convert(params);
ruleHandler = converter.getRuleHandler();
Map<String, FSConfigToCSConfigRuleHandler.RuleAction> actions =
ruleHandler.getActions();
assertEquals("maxCapacityPercentage",
WARNING, actions.get(MAX_CAPACITY_PERCENTAGE));
assertEquals("maxChildCapacity",
WARNING, actions.get(MAX_CHILD_CAPACITY));
assertEquals("userMaxRunningApps",
WARNING, actions.get(USER_MAX_RUNNING_APPS));
assertEquals("userMaxAppsDefault",
WARNING, actions.get(USER_MAX_APPS_DEFAULT));
assertEquals("dynamicMaxAssign",
WARNING, actions.get(DYNAMIC_MAX_ASSIGN));
assertEquals("specifiedNotFirstRule",
WARNING, actions.get(SPECIFIED_NOT_FIRST));
assertEquals("reservationSystem",
WARNING, actions.get(RESERVATION_SYSTEM));
assertEquals("queueAutoCreate",
WARNING, actions.get(QUEUE_AUTO_CREATE));
}
@Test @Test
public void testConvertFSConfigurationUndefinedYarnSiteConfig() public void testConvertFSConfigurationUndefinedYarnSiteConfig()
throws Exception { throws Exception {