From 34cf63c7d0d558bb7baf81c89e031a407b54b9bd Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Mon, 24 Feb 2020 21:45:49 +0100 Subject: [PATCH] YARN-10157. FS-CS converter: initPropertyActions() is not called without rules file. Contributed by Peter Bacsko --- .../FSConfigToCSConfigConverter.java | 4 ++- .../FSConfigToCSConfigRuleHandler.java | 3 +- .../TestFSConfigToCSConfigConverter.java | 36 +++++++++++++++++++ 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java index efc8f0e95c..b0fcc665f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java @@ -166,11 +166,13 @@ private Resource getClusterResource( private void loadConversionRules(String rulesFile) throws IOException { if (rulesFile != null) { LOG.info("Reading conversion rules file from: " + rulesFile); - this.ruleHandler.loadRulesFromFile(rulesFile); + ruleHandler.loadRulesFromFile(rulesFile); } else { LOG.info("Conversion rules file is not defined, " + "using default conversion config!"); } + + ruleHandler.initPropertyActions(); } private Configuration getInputYarnSiteConfig( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigRuleHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigRuleHandler.java index a1db393e4c..3edbd62fd1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigRuleHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigRuleHandler.java @@ -95,7 +95,6 @@ void loadRulesFromFile(String ruleFile) throws IOException { properties.load(is); } actions = new HashMap<>(); - initPropertyActions(); } public FSConfigToCSConfigRuleHandler(ConversionOptions conversionOptions) { @@ -113,7 +112,7 @@ public FSConfigToCSConfigRuleHandler(ConversionOptions conversionOptions) { initPropertyActions(); } - private void initPropertyActions() { + public void initPropertyActions() { setActionForProperty(MAX_CAPACITY_PERCENTAGE); setActionForProperty(MAX_CHILD_CAPACITY); setActionForProperty(USER_MAX_RUNNING_APPS); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java index c229d94915..576025858d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java @@ -26,6 +26,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.USER_MAX_APPS_DEFAULT; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.USER_MAX_RUNNING_APPS; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.RuleAction.ABORT; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.RuleAction.WARNING; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -372,6 +373,41 @@ public void testConvertFSConfigurationRulesFile() throws Exception { ABORT, actions.get(QUEUE_AUTO_CREATE)); } + @Test + public void testConvertFSConfigurationWithoutRulesFile() throws Exception { + ruleHandler = new FSConfigToCSConfigRuleHandler( + createDefaultConversionOptions()); + createConverter(); + + FSConfigToCSConfigConverterParams params = + createDefaultParamsBuilder() + .withClusterResource(CLUSTER_RESOURCE_STRING) + .build(); + + converter.convert(params); + + ruleHandler = converter.getRuleHandler(); + Map actions = + ruleHandler.getActions(); + + assertEquals("maxCapacityPercentage", + WARNING, actions.get(MAX_CAPACITY_PERCENTAGE)); + assertEquals("maxChildCapacity", + WARNING, actions.get(MAX_CHILD_CAPACITY)); + assertEquals("userMaxRunningApps", + WARNING, actions.get(USER_MAX_RUNNING_APPS)); + assertEquals("userMaxAppsDefault", + WARNING, actions.get(USER_MAX_APPS_DEFAULT)); + assertEquals("dynamicMaxAssign", + WARNING, actions.get(DYNAMIC_MAX_ASSIGN)); + assertEquals("specifiedNotFirstRule", + WARNING, actions.get(SPECIFIED_NOT_FIRST)); + assertEquals("reservationSystem", + WARNING, actions.get(RESERVATION_SYSTEM)); + assertEquals("queueAutoCreate", + WARNING, actions.get(QUEUE_AUTO_CREATE)); + } + @Test public void testConvertFSConfigurationUndefinedYarnSiteConfig() throws Exception {