From c0d084247c0d1b7b701bc82e8847096912e5d8a0 Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Tue, 3 Mar 2020 09:46:32 +0100 Subject: [PATCH] YARN-10175. FS-CS converter: only convert placement rules if a cmd line switch is defined. Contributed by Peter Bacsko --- .../FSConfigToCSConfigArgumentHandler.java | 7 +++ .../FSConfigToCSConfigConverter.java | 35 +++++++++++---- .../FSConfigToCSConfigConverterParams.java | 17 +++++++- ...TestFSConfigToCSConfigArgumentHandler.java | 43 +++++++++++++++++++ .../TestFSConfigToCSConfigConverter.java | 39 +++++++++++++++++ 5 files changed, 131 insertions(+), 10 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java index 9121a6da9f..4c3ec31c0f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java @@ -98,6 +98,10 @@ public enum CliOption { "Disables checking whether a placement rule is terminal to maintain" + " backward compatibility with configs that were made before YARN-8967.", false), + CONVERT_PLACEMENT_RULES("convert placement rules", + "m", "convert-placement-rules", + "Convert Fair Scheduler placement rules to Capacity" + + " Scheduler mapping rules", false), HELP("help", "h", "help", "Displays the list of options", false); private final String name; @@ -216,6 +220,8 @@ private FSConfigToCSConfigConverterParams validateInputFiles( cliParser.getOptionValue(CliOption.CONVERSION_RULES.shortSwitch); String outputDir = cliParser.getOptionValue(CliOption.OUTPUT_DIR.shortSwitch); + boolean convertPlacementRules = + cliParser.hasOption(CliOption.CONVERT_PLACEMENT_RULES.shortSwitch); checkFile(CliOption.YARN_SITE, yarnSiteXmlFile); checkFile(CliOption.FAIR_SCHEDULER, fairSchedulerXmlFile); @@ -231,6 +237,7 @@ private FSConfigToCSConfigConverterParams validateInputFiles( cliParser.getOptionValue(CliOption.CLUSTER_RESOURCE.shortSwitch)) .withConsole(cliParser.hasOption(CliOption.CONSOLE_MODE.shortSwitch)) .withOutputDirectory(outputDir) + .withConvertPlacementRules(convertPlacementRules) .build(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java index b0fcc665f3..ec8178b03a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java @@ -91,10 +91,12 @@ public class FSConfigToCSConfigConverter { private Configuration convertedYarnSiteConfig; private Configuration capacitySchedulerConfig; private FSConfigToCSConfigRuleHandler ruleHandler; + private QueuePlacementConverter placementConverter; private OutputStream yarnSiteOutputStream; private OutputStream capacitySchedulerOutputStream; private boolean consoleMode = false; + private boolean convertPlacementRules = false; public FSConfigToCSConfigConverter(FSConfigToCSConfigRuleHandler ruleHandler, ConversionOptions conversionOptions) { @@ -102,6 +104,7 @@ public FSConfigToCSConfigConverter(FSConfigToCSConfigRuleHandler this.conversionOptions = conversionOptions; this.yarnSiteOutputStream = System.out; this.capacitySchedulerOutputStream = System.out; + this.placementConverter = new QueuePlacementConverter(); } public void convert(FSConfigToCSConfigConverterParams params) @@ -113,6 +116,8 @@ public void convert(FSConfigToCSConfigConverterParams params) handleFairSchedulerConfig(params, inputYarnSiteConfig); this.clusterResource = getClusterResource(params); + this.convertPlacementRules = params.isConvertPlacementRules(); + convert(inputYarnSiteConfig); } @@ -309,16 +314,19 @@ private void convertCapacitySchedulerXml(FairScheduler fs) { queueConverter.convertQueueHierarchy(rootQueue); emitACLs(fs); - PlacementManager placementManager = - fs.getRMContext().getQueuePlacementManager(); + if (convertPlacementRules) { + LOG.info("Converting placement rules"); + PlacementManager placementManager = + fs.getRMContext().getQueuePlacementManager(); - if (placementManager.getPlacementRules().size() > 0) { - QueuePlacementConverter placementConverter = - new QueuePlacementConverter(); - Map properties = - placementConverter.convertPlacementPolicy(placementManager, - ruleHandler, userAsDefaultQueue); - properties.forEach((k, v) -> capacitySchedulerConfig.set(k, v)); + if (placementManager.getPlacementRules().size() > 0) { + Map properties = + placementConverter.convertPlacementPolicy(placementManager, + ruleHandler, userAsDefaultQueue); + properties.forEach((k, v) -> capacitySchedulerConfig.set(k, v)); + } + } else { + LOG.info("Ignoring the conversion of placement rules"); } } @@ -432,6 +440,15 @@ Configuration getYarnSiteConfig() { return convertedYarnSiteConfig; } + @VisibleForTesting + void setConvertPlacementRules(boolean convertPlacementRules) { + this.convertPlacementRules = convertPlacementRules; + } + + @VisibleForTesting + void setPlacementConverter(QueuePlacementConverter converter) { + this.placementConverter = converter; + } /* * Determines whether is present * in the allocation file or not. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverterParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverterParams.java index ca2eb7ea38..207316445f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverterParams.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverterParams.java @@ -27,6 +27,9 @@ public final class FSConfigToCSConfigConverterParams { private boolean console; private String clusterResource; private String outputDirectory; + private boolean convertPlacementRules; + + private FSConfigToCSConfigConverterParams() { //must use builder @@ -56,6 +59,10 @@ public String getOutputDirectory() { return outputDirectory; } + public boolean isConvertPlacementRules() { + return convertPlacementRules; + } + @Override public String toString() { return "FSConfigToCSConfigConverterParams{" + @@ -63,7 +70,8 @@ public String toString() { ", fairSchedulerXmlConfig='" + fairSchedulerXmlConfig + '\'' + ", conversionRulesConfig='" + conversionRulesConfig + '\'' + ", clusterResource='" + clusterResource + '\'' + - ", console=" + console + + ", console=" + console + '\'' + + ", convertPlacementRules=" + convertPlacementRules + '}'; } @@ -78,6 +86,7 @@ public static final class Builder { private boolean console; private String clusterResource; private String outputDirectory; + private boolean convertPlacementRules; private Builder() { } @@ -116,6 +125,11 @@ public Builder withOutputDirectory(String outputDir) { return this; } + public Builder withConvertPlacementRules(boolean convertPlacementRules) { + this.convertPlacementRules = convertPlacementRules; + return this; + } + public FSConfigToCSConfigConverterParams build() { FSConfigToCSConfigConverterParams params = new FSConfigToCSConfigConverterParams(); @@ -125,6 +139,7 @@ public FSConfigToCSConfigConverterParams build() { params.yarnSiteXmlConfig = this.yarnSiteXmlConfig; params.conversionRulesConfig = this.conversionRulesConfig; params.outputDirectory = this.outputDirectory; + params.convertPlacementRules = this.convertPlacementRules; return params; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java index 7a29c7145c..966600fc78 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java @@ -19,6 +19,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.verify; import java.io.File; import java.io.IOException; @@ -555,4 +556,46 @@ public void testCapacitySchedulerXmlExistsInOutputFolder() testFileExistsInOutputFolder( YarnConfiguration.CS_CONFIGURATION_FILE); } + + @Test + public void testPlacementRulesConversionEnabled() throws Exception { + testPlacementRuleConversion(true); + } + + @Test + public void testPlacementRulesConversionDisabled() throws Exception { + testPlacementRuleConversion(false); + } + + private void testPlacementRuleConversion(boolean enabled) throws Exception { + setupFSConfigConversionFiles(true); + + String[] args = null; + if (enabled) { + args = getArgumentsAsArrayWithDefaults("-f", + FSConfigConverterTestCommons.FS_ALLOC_FILE, + "-p", "-m"); + } else { + args = getArgumentsAsArrayWithDefaults("-f", + FSConfigConverterTestCommons.FS_ALLOC_FILE, + "-p"); + } + FSConfigToCSConfigArgumentHandler argumentHandler = + new FSConfigToCSConfigArgumentHandler(conversionOptions); + argumentHandler.setConverterSupplier(this::getMockConverter); + + argumentHandler.parseAndConvert(args); + + ArgumentCaptor captor = + ArgumentCaptor.forClass(FSConfigToCSConfigConverterParams.class); + verify(mockConverter).convert(captor.capture()); + FSConfigToCSConfigConverterParams params = captor.getValue(); + + if (enabled) { + assertTrue("-m switch had no effect", params.isConvertPlacementRules()); + } else { + assertFalse("Placement rule conversion was enabled", + params.isConvertPlacementRules()); + } + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java index 576025858d..5c04510fd9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java @@ -31,6 +31,9 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -43,6 +46,7 @@ import org.apache.hadoop.service.ServiceStateException; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; @@ -86,6 +90,9 @@ public class TestFSConfigToCSConfigConverter { @Mock private DryRunResultHolder dryRunResultHolder; + @Mock + private QueuePlacementConverter placementConverter; + private FSConfigToCSConfigConverter converter; private Configuration config; @@ -583,6 +590,7 @@ private void testUserAsDefaultQueueWithoutPlacementRules(boolean config.setBoolean(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, userAsDefaultQueue); + converter.setConvertPlacementRules(true); converter.convert(config); Configuration convertedConf = getConvertedCSConfig(); @@ -650,6 +658,37 @@ private void testAutoCreateChildQueuesWithoutPlacementRules( } } + @Test + public void testPlacementRulesConversionDisabled() throws Exception { + FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder() + .withClusterResource(CLUSTER_RESOURCE_STRING) + .withFairSchedulerXmlConfig(FAIR_SCHEDULER_XML) + .withConvertPlacementRules(false) + .build(); + + converter.setPlacementConverter(placementConverter); + converter.convert(params); + + verifyZeroInteractions(placementConverter); + } + + @Test + public void testPlacementRulesConversionEnabled() throws Exception { + FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder() + .withClusterResource(CLUSTER_RESOURCE_STRING) + .withFairSchedulerXmlConfig(FAIR_SCHEDULER_XML) + .withConvertPlacementRules(true) + .build(); + + converter.setPlacementConverter(placementConverter); + converter.convert(params); + + verify(placementConverter).convertPlacementPolicy( + any(PlacementManager.class), + any(FSConfigToCSConfigRuleHandler.class), + any(Boolean.class)); + } + private Configuration getConvertedCSConfig() { ByteArrayInputStream input = new ByteArrayInputStream(csConfigOut.toByteArray());