From ca297680352b7bf20ca6cd98ff81bb68e1358c5b Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Tue, 28 Jan 2020 15:22:12 +0100 Subject: [PATCH] YARN-10085. FS-CS converter: remove mixed ordering policy check. Contributed by Peter Bacsko --- .../FSConfigToCSConfigConverter.java | 61 +++++++++--- .../FSConfigToCSConfigRuleHandler.java | 12 +++ .../fair/converter/FSQueueConverter.java | 27 ++---- .../converter/FSQueueConverterBuilder.java | 6 ++ .../fair/converter/FSYarnSiteConverter.java | 9 +- .../converter/QueuePlacementConverter.java | 3 - .../TestFSConfigToCSConfigConverter.java | 67 ++++++++----- .../TestFSConfigToCSConfigRuleHandler.java | 5 + .../fair/converter/TestFSQueueConverter.java | 20 ++++ .../converter/TestFSYarnSiteConverter.java | 33 +++++-- .../fair-scheduler-onlyfairpolicy.xml | 94 +++++++++++++++++++ .../fair-scheduler-orderingpolicy-mixed.xml | 8 +- 12 files changed, 277 insertions(+), 68 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-onlyfairpolicy.xml diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java index a8fdb518e3..eae952d438 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java @@ -23,6 +23,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; +import java.util.List; import java.util.Map; import org.apache.hadoop.conf.Configuration; @@ -39,9 +40,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfigurationException; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.ConfigurableResource; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSParentQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; -import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -73,6 +75,7 @@ public class FSConfigToCSConfigConverter { private boolean sizeBasedWeight = false; private boolean userAsDefaultQueue = false; private ConversionOptions conversionOptions; + private boolean drfUsed = false; private Configuration yarnSiteConfig; private Configuration capacitySchedulerConfig; @@ -198,7 +201,7 @@ private void handleFairSchedulerConfig( @VisibleForTesting void convert(Configuration conf) throws Exception { System.out.println(WARNING_TEXT); - + // initialize Fair Scheduler RMContext ctx = new RMContextImpl(); PlacementManager placementManager = new PlacementManager(); @@ -208,6 +211,8 @@ void convert(Configuration conf) throws Exception { fs.setRMContext(ctx); fs.init(conf); + drfUsed = isDrfUsed(fs); + AllocationConfiguration allocConf = fs.getAllocationConfiguration(); queueMaxAppsDefault = allocConf.getQueueMaxAppsDefault(); queueMaxAMShareDefault = allocConf.getQueueMaxAMShareDefault(); @@ -246,7 +251,7 @@ void setCapacitySchedulerConfigOutputStream(OutputStream out) { private void convertYarnSiteXml(Configuration conf) { FSYarnSiteConverter siteConverter = new FSYarnSiteConverter(); - siteConverter.convertSiteProperties(conf, yarnSiteConfig); + siteConverter.convertSiteProperties(conf, yarnSiteConfig, drfUsed); autoCreateChildQueues = siteConverter.isAutoCreateChildQueues(); preemptionEnabled = siteConverter.isPreemptionEnabled(); @@ -271,6 +276,7 @@ private void convertCapacitySchedulerXml(FairScheduler fs) { .withQueueMaxAMShareDefault(queueMaxAMShareDefault) .withQueueMaxAppsDefault(queueMaxAppsDefault) .withConversionOptions(conversionOptions) + .withDrfUsed(drfUsed) .build(); queueConverter.convertQueueHierarchy(rootQueue); @@ -287,18 +293,6 @@ private void convertCapacitySchedulerXml(FairScheduler fs) { ruleHandler, userAsDefaultQueue); properties.forEach((k, v) -> capacitySchedulerConfig.set(k, v)); } - - // Validate ordering policy - if (queueConverter.isDrfPolicyUsedOnQueueLevel()) { - if (queueConverter.isFifoOrFairSharePolicyUsed()) { - throw new ConversionException( - "DRF ordering policy cannot be used together with fifo/fair"); - } else { - capacitySchedulerConfig.set( - CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, - DominantResourceCalculator.class.getCanonicalName()); - } - } } private void emitDefaultMaxApplications() { @@ -359,6 +353,38 @@ private void checkUserMaxAppsDefault(AllocationConfiguration allocConf) { } } + private boolean isDrfUsed(FairScheduler fs) { + FSQueue rootQueue = fs.getQueueManager().getRootQueue(); + AllocationConfiguration allocConf = fs.getAllocationConfiguration(); + + String defaultPolicy = allocConf.getDefaultSchedulingPolicy().getName(); + + if (DominantResourceFairnessPolicy.NAME.equals(defaultPolicy)) { + return true; + } else { + return isDrfUsedOnQueueLevel(rootQueue); + } + } + + private boolean isDrfUsedOnQueueLevel(FSQueue queue) { + String policy = queue.getPolicy().getName(); + boolean usesDrf = DominantResourceFairnessPolicy.NAME.equals(policy); + + if (usesDrf) { + return true; + } else { + List children = queue.getChildQueues(); + + if (children != null) { + for (FSQueue child : children) { + usesDrf |= isDrfUsedOnQueueLevel(child); + } + } + + return usesDrf; + } + } + @VisibleForTesting Resource getClusterResource() { return clusterResource; @@ -373,4 +399,9 @@ public void setClusterResource(Resource clusterResource) { FSConfigToCSConfigRuleHandler getRuleHandler() { return ruleHandler; } + + @VisibleForTesting + Configuration getYarnSiteConfig() { + return yarnSiteConfig; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigRuleHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigRuleHandler.java index 2088334513..a1db393e4c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigRuleHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigRuleHandler.java @@ -73,6 +73,9 @@ public class FSConfigToCSConfigRuleHandler { public static final String QUEUE_AUTO_CREATE = "queueAutoCreate.action"; + public static final String FAIR_AS_DRF = + "fairAsDrf.action"; + @VisibleForTesting enum RuleAction { WARNING, @@ -119,6 +122,7 @@ private void initPropertyActions() { setActionForProperty(SPECIFIED_NOT_FIRST); setActionForProperty(RESERVATION_SYSTEM); setActionForProperty(QUEUE_AUTO_CREATE); + setActionForProperty(FAIR_AS_DRF); } public void handleMaxCapacityPercentage(String queueName) { @@ -181,6 +185,14 @@ public void handleQueueAutoCreate(String placementRule) { placementRule)); } + public void handleFairAsDrf(String queueName) { + handle(FAIR_AS_DRF, + null, + format( + "Queue %s will use DRF policy instead of Fair", + queueName)); + } + private void handle(String actionName, String fsSetting, String message) { RuleAction action = actions.get(actionName); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java index 1722302dc2..44bdad48bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java @@ -55,9 +55,7 @@ public class FSQueueConverter { private final float queueMaxAMShareDefault; private final boolean autoCreateChildQueues; private final int queueMaxAppsDefault; - - private boolean fifoOrFairSharePolicyUsed; - private boolean drfPolicyUsedOnQueueLevel; + private final boolean drfUsed; private ConversionOptions conversionOptions; @@ -72,6 +70,7 @@ public FSQueueConverter(FSQueueConverterBuilder builder) { this.autoCreateChildQueues = builder.autoCreateChildQueues; this.queueMaxAppsDefault = builder.queueMaxAppsDefault; this.conversionOptions = builder.conversionOptions; + this.drfUsed = builder.drfUsed; } public void convertQueueHierarchy(FSQueue queue) { @@ -105,14 +104,6 @@ public void convertQueueHierarchy(FSQueue queue) { } } - public boolean isFifoOrFairSharePolicyUsed() { - return fifoOrFairSharePolicyUsed; - } - - public boolean isDrfPolicyUsedOnQueueLevel() { - return drfPolicyUsedOnQueueLevel; - } - /** * Generates yarn.scheduler.capacity.<queue-name>.queues. * @param queueName @@ -306,20 +297,20 @@ private void emitOrderingPolicy(String queueName, FSQueue queue) { String policy = queue.getPolicy().getName(); switch (policy) { + case DominantResourceFairnessPolicy.NAME: + capacitySchedulerConfig.set(PREFIX + queueName + + ".ordering-policy", FairSharePolicy.NAME); + break; case FairSharePolicy.NAME: capacitySchedulerConfig.set(PREFIX + queueName + ".ordering-policy", FairSharePolicy.NAME); - fifoOrFairSharePolicyUsed = true; + if (drfUsed) { + ruleHandler.handleFairAsDrf(queueName); + } break; case FifoPolicy.NAME: capacitySchedulerConfig.set(PREFIX + queueName + ".ordering-policy", FifoPolicy.NAME); - fifoOrFairSharePolicyUsed = true; - break; - case DominantResourceFairnessPolicy.NAME: - // DRF is not supported on a queue level, - // it has to be global - drfPolicyUsedOnQueueLevel = true; break; default: String msg = String.format("Unexpected ordering policy " + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverterBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverterBuilder.java index 615f1b946c..dd5f05cd64 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverterBuilder.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverterBuilder.java @@ -32,6 +32,7 @@ public final class FSQueueConverterBuilder { float queueMaxAMShareDefault; int queueMaxAppsDefault; ConversionOptions conversionOptions; + boolean drfUsed; private FSQueueConverterBuilder() { } @@ -94,6 +95,11 @@ public FSQueueConverterBuilder withConversionOptions( return this; } + public FSQueueConverterBuilder withDrfUsed(boolean drfUsed) { + this.drfUsed = drfUsed; + return this; + } + public FSQueueConverter build() { return new FSQueueConverter(this); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java index f6d023dbe9..25db4d0c6b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java @@ -23,6 +23,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; +import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; /** * Converts a Fair Scheduler site configuration to Capacity Scheduler @@ -37,7 +38,7 @@ public class FSYarnSiteConverter { @SuppressWarnings({"deprecation", "checkstyle:linelength"}) public void convertSiteProperties(Configuration conf, - Configuration yarnSiteConfig) { + Configuration yarnSiteConfig, boolean drfUsed) { yarnSiteConfig.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getCanonicalName()); @@ -139,6 +140,12 @@ public void convertSiteProperties(Configuration conf, FairSchedulerConfiguration.DEFAULT_USER_AS_DEFAULT_QUEUE)) { userAsDefaultQueue = true; } + + if (drfUsed) { + yarnSiteConfig.set( + CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, + DominantResourceCalculator.class.getCanonicalName()); + } } public boolean isPreemptionEnabled() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java index 4641dfcdc3..8d3182f38a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java @@ -82,7 +82,6 @@ Map convertPlacementPolicy(PlacementManager placementManager, } mapping.append("u:" + USER + ":").append(defaultRule.defaultQueueName); } else if (rule instanceof SecondaryGroupExistingPlacementRule) { - // TODO: wait for YARN-9840 if (mapping.length() > 0) { mapping.append(";"); } @@ -107,10 +106,8 @@ private void handleNestedRule(StringBuilder mapping, mapping.append(";"); } if (pr instanceof PrimaryGroupPlacementRule) { - // TODO: wait for YARN-9841 mapping.append("u:" + USER + ":" + PRIMARY_GROUP + "." + USER); } else if (pr instanceof SecondaryGroupExistingPlacementRule) { - // TODO: wait for YARN-9865 mapping.append("u:" + USER + ":" + SECONDARY_GROUP + "." + USER); } else if (pr instanceof DefaultPlacementRule) { DefaultPlacementRule defaultRule = (DefaultPlacementRule) pr; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java index 4525dae2d5..dc51db923f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java @@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; +import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -60,6 +61,8 @@ */ @RunWith(MockitoJUnitRunner.class) public class TestFSConfigToCSConfigConverter { + private static final String CLUSTER_RESOURCE_STRING = + "vcores=20, memory-mb=240"; private static final Resource CLUSTER_RESOURCE = Resource.newInstance(16384, 16); private static final String FILE_PREFIX = "file:"; @@ -67,6 +70,10 @@ public class TestFSConfigToCSConfigConverter { prepareFileName("fair-scheduler-conversion.xml"); private static final String FS_INVALID_PLACEMENT_RULES_XML = prepareFileName("fair-scheduler-invalidplacementrules.xml"); + private static final String FS_ONLY_FAIR_POLICY_XML = + prepareFileName("fair-scheduler-onlyfairpolicy.xml"); + private static final String FS_MIXED_POLICY_XML = + prepareFileName("fair-scheduler-orderingpolicy-mixed.xml"); @Mock private FSConfigToCSConfigRuleHandler ruleHandler; @@ -215,20 +222,6 @@ public void testDefaultMaxRunningApps() throws Exception { conf.getInt(PREFIX + "maximum-applications", -1)); } - @Test - public void testMixedQueueOrderingPolicy() throws Exception { - expectedException.expect(ConversionException.class); - expectedException.expectMessage( - "DRF ordering policy cannot be used together with fifo/fair"); - String absolutePath = - new File("src/test/resources/fair-scheduler-orderingpolicy-mixed.xml") - .getAbsolutePath(); - config.set(FairSchedulerConfiguration.ALLOCATION_FILE, - FILE_PREFIX + absolutePath); - - converter.convert(config); - } - @Test public void testQueueMaxChildCapacityNotSupported() throws Exception { expectedException.expect(UnsupportedPropertyException.class); @@ -277,7 +270,7 @@ public void testUserMaxAppsDefaultNotSupported() throws Exception { @Test public void testConvertFSConfigurationClusterResource() throws Exception { FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder() - .withClusterResource("vcores=20, memory-mb=240") + .withClusterResource(CLUSTER_RESOURCE_STRING) .build(); converter.convert(params); assertEquals("Resource", Resource.newInstance(240, 20), @@ -288,7 +281,7 @@ public void testConvertFSConfigurationClusterResource() throws Exception { public void testConvertFSConfigPctModeUsedAndClusterResourceDefined() throws Exception { FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder() - .withClusterResource("vcores=20, memory-mb=240") + .withClusterResource(CLUSTER_RESOURCE_STRING) .build(); converter.convert(params); assertEquals("Resource", Resource.newInstance(240, 20), @@ -394,7 +387,7 @@ public void testConvertFSConfigurationUndefinedYarnSiteConfig() @Test public void testConvertCheckOutputDir() throws Exception { FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder() - .withClusterResource("vcores=20, memory-mb=240") + .withClusterResource(CLUSTER_RESOURCE_STRING) .build(); converter.convert(params); @@ -419,7 +412,7 @@ public void testFairSchedulerXmlIsNotDefinedNeitherDirectlyNorInYarnSiteXml() throws Exception { FSConfigToCSConfigConverterParams params = createParamsBuilder(YARN_SITE_XML_NO_REF_TO_FS_XML) - .withClusterResource("vcores=20, memory-mb=240") + .withClusterResource(CLUSTER_RESOURCE_STRING) .build(); expectedException.expect(PreconditionException.class); @@ -430,7 +423,7 @@ public void testFairSchedulerXmlIsNotDefinedNeitherDirectlyNorInYarnSiteXml() @Test public void testInvalidFairSchedulerXml() throws Exception { FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder() - .withClusterResource("vcores=20, memory-mb=240") + .withClusterResource(CLUSTER_RESOURCE_STRING) .withFairSchedulerXmlConfig(FAIR_SCHEDULER_XML_INVALID) .build(); @@ -442,7 +435,7 @@ public void testInvalidFairSchedulerXml() throws Exception { public void testInvalidYarnSiteXml() throws Exception { FSConfigToCSConfigConverterParams params = createParamsBuilder(YARN_SITE_XML_INVALID) - .withClusterResource("vcores=20, memory-mb=240") + .withClusterResource(CLUSTER_RESOURCE_STRING) .build(); expectedException.expect(RuntimeException.class); @@ -464,7 +457,7 @@ public void testConversionWithInvalidPlacementRules() throws Exception { public void testConversionWhenInvalidPlacementRulesIgnored() throws Exception { FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder() - .withClusterResource("vcores=20, memory-mb=240") + .withClusterResource(CLUSTER_RESOURCE_STRING) .withFairSchedulerXmlConfig(FS_INVALID_PLACEMENT_RULES_XML) .build(); @@ -479,6 +472,38 @@ public void testConversionWhenInvalidPlacementRulesIgnored() // expected: no exception } + @Test + public void testConversionWhenOnlyFairPolicyIsUsed() throws Exception { + FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder() + .withClusterResource(CLUSTER_RESOURCE_STRING) + .withFairSchedulerXmlConfig(FS_ONLY_FAIR_POLICY_XML) + .build(); + + converter.convert(params); + + Configuration convertedConfig = converter.getYarnSiteConfig(); + + assertEquals("Resource calculator class shouldn't be set", null, + convertedConfig.getClass( + CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, null)); + } + + @Test + public void testConversionWhenMixedPolicyIsUsed() throws Exception { + FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder() + .withClusterResource(CLUSTER_RESOURCE_STRING) + .withFairSchedulerXmlConfig(FS_MIXED_POLICY_XML) + .build(); + + converter.convert(params); + + Configuration convertedConfig = converter.getYarnSiteConfig(); + + assertEquals("Resource calculator type", DominantResourceCalculator.class, + convertedConfig.getClass( + CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, null)); + } + private Configuration getConvertedCSConfig() { ByteArrayInputStream input = new ByteArrayInputStream(csConfigOut.toByteArray()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigRuleHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigRuleHandler.java index b6caadd5b2..4539e1b1a4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigRuleHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigRuleHandler.java @@ -25,6 +25,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.SPECIFIED_NOT_FIRST; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.USER_MAX_APPS_DEFAULT; import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.USER_MAX_RUNNING_APPS; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.FAIR_AS_DRF; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -86,6 +87,7 @@ public void testAllRulesWarning() throws IOException { rules.put(SPECIFIED_NOT_FIRST, WARNING); rules.put(USER_MAX_APPS_DEFAULT, WARNING); rules.put(USER_MAX_RUNNING_APPS, WARNING); + rules.put(FAIR_AS_DRF, WARNING); ruleHandler = new FSConfigToCSConfigRuleHandler(rules, createDefaultConversionOptions()); @@ -111,6 +113,8 @@ public void testAllRulesAbort() throws IOException { rules.put(SPECIFIED_NOT_FIRST, ABORT); rules.put(USER_MAX_APPS_DEFAULT, ABORT); rules.put(USER_MAX_RUNNING_APPS, ABORT); + rules.put(USER_MAX_RUNNING_APPS, ABORT); + rules.put(FAIR_AS_DRF, ABORT); rules.put(MAX_CHILD_QUEUE_LIMIT, "1"); ruleHandler = new FSConfigToCSConfigRuleHandler(rules, @@ -126,6 +130,7 @@ public void testAllRulesAbort() throws IOException { expectAbort(() -> ruleHandler.handleSpecifiedNotFirstRule()); expectAbort(() -> ruleHandler.handleUserMaxApps()); expectAbort(() -> ruleHandler.handleUserMaxAppsDefault()); + expectAbort(() -> ruleHandler.handleFairAsDrf("test")); } @Test(expected = ConversionException.class) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java index 3507c83545..22213647bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java @@ -20,6 +20,9 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import java.io.File; import java.io.IOException; @@ -400,6 +403,23 @@ public void testQueueOrderingPolicy() throws Exception { csConfig.get(PREFIX + "root.admins.bob.ordering-policy")); } + @Test + public void testQueueUnsupportedMixedOrderingPolicy() throws IOException { + converter = builder.withDrfUsed(true).build(); + String absolutePath = + new File("src/test/resources/fair-scheduler-orderingpolicy-mixed.xml") + .getAbsolutePath(); + config.set(FairSchedulerConfiguration.ALLOCATION_FILE, + FILE_PREFIX + absolutePath); + fs.close(); + fs = createFairScheduler(); + rootQueue = fs.getQueueManager().getRootQueue(); + + converter.convertQueueHierarchy(rootQueue); + + verify(ruleHandler, times(6)).handleFairAsDrf(anyString()); + } + @Test public void testQueueMaxChildCapacityNotSupported() { converter = builder.build(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java index edb9a4e9ca..0bd8fb1dbb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java @@ -20,6 +20,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; +import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; +import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; import org.junit.Before; import org.junit.Test; @@ -50,7 +52,7 @@ public void testSiteContinuousSchedulingConversion() { yarnConfig.setInt( FairSchedulerConfiguration.CONTINUOUS_SCHEDULING_SLEEP_MS, 666); - converter.convertSiteProperties(yarnConfig, yarnConvertedConfig); + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false); assertTrue("Cont. scheduling", yarnConvertedConfig.getBoolean( CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE, false)); @@ -65,7 +67,7 @@ public void testSiteMinimumAllocationIncrementConversion() { yarnConfig.setInt("yarn.resource-types.memory-mb.increment-allocation", 11); yarnConfig.setInt("yarn.resource-types.vcores.increment-allocation", 5); - converter.convertSiteProperties(yarnConfig, yarnConvertedConfig); + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false); assertEquals("Memory alloc increment", 11, yarnConvertedConfig.getInt("yarn.scheduler.minimum-allocation-mb", @@ -83,7 +85,7 @@ public void testSitePreemptionConversion() { FairSchedulerConfiguration.WAIT_TIME_BEFORE_NEXT_STARVATION_CHECK_MS, 321); - converter.convertSiteProperties(yarnConfig, yarnConvertedConfig); + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false); assertTrue("Preemption enabled", yarnConvertedConfig.getBoolean( @@ -103,7 +105,7 @@ public void testSitePreemptionConversion() { public void testSiteAssignMultipleConversion() { yarnConfig.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, true); - converter.convertSiteProperties(yarnConfig, yarnConvertedConfig); + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false); assertTrue("Assign multiple", yarnConvertedConfig.getBoolean( @@ -115,7 +117,7 @@ public void testSiteAssignMultipleConversion() { public void testSiteMaxAssignConversion() { yarnConfig.setInt(FairSchedulerConfiguration.MAX_ASSIGN, 111); - converter.convertSiteProperties(yarnConfig, yarnConvertedConfig); + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false); assertEquals("Max assign", 111, yarnConvertedConfig.getInt( @@ -129,7 +131,7 @@ public void testSiteLocalityThresholdConversion() { yarnConfig.set(FairSchedulerConfiguration.LOCALITY_THRESHOLD_RACK, "321.321"); - converter.convertSiteProperties(yarnConfig, yarnConvertedConfig); + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false); assertEquals("Locality threshold node", "123.123", yarnConvertedConfig.get( @@ -138,4 +140,23 @@ public void testSiteLocalityThresholdConversion() { yarnConvertedConfig.get( CapacitySchedulerConfiguration.RACK_LOCALITY_ADDITIONAL_DELAY)); } + + @Test + public void testSiteDrfEnabledConversion() { + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, true); + + assertEquals("Resource calculator type", DominantResourceCalculator.class, + yarnConvertedConfig.getClass( + CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, null)); + } + + @Test + public void testSiteDrfDisabledConversion() { + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig, false); + + assertEquals("Resource calculator type", DefaultResourceCalculator.class, + yarnConvertedConfig.getClass( + CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, + CapacitySchedulerConfiguration.DEFAULT_RESOURCE_CALCULATOR_CLASS)); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-onlyfairpolicy.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-onlyfairpolicy.xml new file mode 100644 index 0000000000..e03be5f092 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-onlyfairpolicy.xml @@ -0,0 +1,94 @@ + + + + + 1.0 + fair + alice,bob,joe,john hadoop_users + alice,bob,joe,john hadoop_users + + 1.0 + fair + + + 1.0 + fair + + 1.0 + fair + john + john + vcores=2,memory-mb=8192 + + + memory-mb=50.0%, vcores=50.0% + 3.0 + false + fair + joe + joe + + + + memory-mb=8192, vcores=1 + 1.0 + fair + vcores=3,memory-mb=4096 + + memory-mb=16384, vcores=4 + 2 + 3.0 + false + fair + alice + alice + 0.15 + memory-mb=16384, vcores=4 + + + memory-mb=8192, vcores=2 + 1.0 + fair + bob + bob + -1.0 + + + + + 30 + + 10 + 23 + 24 + 0.12 + 15 + fair + 0.16 + + + + + + + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-orderingpolicy-mixed.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-orderingpolicy-mixed.xml index 3a2a59300c..778fb869e9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-orderingpolicy-mixed.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-orderingpolicy-mixed.xml @@ -31,7 +31,7 @@ drf 1.0 - drf + fair john john vcores=2,memory-mb=8192 @@ -40,7 +40,7 @@ memory-mb=50.0%, vcores=50.0% 3.0 false - drf + fair joe joe @@ -55,7 +55,7 @@ 2 3.0 false - drf + fair alice alice 0.15 @@ -63,7 +63,7 @@ memory-mb=8192, vcores=2 1.0 - drf + fair bob bob -1.0