diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index d939ac1525..351519b0b6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -21,6 +21,10 @@ import com.google.common.annotations.VisibleForTesting; import com.sun.jersey.spi.container.servlet.ServletContainer; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigArgumentHandler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigArgumentHandler.CliOption; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigConverter; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.Marker; @@ -136,7 +140,6 @@ import java.net.InetSocketAddress; import java.net.URI; import java.net.URL; -import java.net.URLClassLoader; import java.nio.charset.Charset; import java.security.PrivilegedExceptionAction; import java.security.SecureRandom; @@ -226,6 +229,13 @@ public class ResourceManager extends CompositeService private Configuration conf; private UserGroupInformation rmLoginUGI; + private static FSConfigToCSConfigArgumentHandler + fsConfigConversionArgumentHandler; + + static { + FSConfigToCSConfigConverter converter = initFSConfigConverter(); + initFSArgumentHandler(converter); + } public ResourceManager() { super("ResourceManager"); @@ -1556,6 +1566,22 @@ public static void main(String argv[]) { } else if (argv[0].equals("-remove-application-from-state-store") && argv.length == 2) { removeApplication(conf, argv[1]); + } else if (argv[0].equals("-convert-fs-configuration")) { + String[] args = Arrays.copyOfRange(argv, 1, argv.length); + try { + int exitCode = + fsConfigConversionArgumentHandler.parseAndConvert(args); + if (exitCode != 0) { + LOG.error(FATAL, + "Error while starting FS configuration conversion, " + + "see previous error messages for details!"); + System.exit(exitCode); + } + } catch (Throwable t) { + LOG.error(FATAL, + "Error while starting FS configuration conversion!", t); + System.exit(-1); + } } else { printUsage(System.err); } @@ -1666,6 +1692,13 @@ private static void printUsage(PrintStream out) { out.println("Usage: yarn resourcemanager [-format-state-store]"); out.println(" " + "[-remove-application-from-state-store ]" + "\n"); + + out.println("[-convert-fs-configuration "); + out.println(FSConfigToCSConfigConverter.WARNING_TEXT); + for (CliOption cliOption : CliOption.values()) { + out.println(" " + cliOption.getAsArgumentString()); + } + out.println("]"); } protected RMAppLifetimeMonitor createRMAppLifetimeMonitor() { @@ -1683,4 +1716,17 @@ private void registerMXBean() { public boolean isSecurityEnabled() { return UserGroupInformation.isSecurityEnabled(); } + + @VisibleForTesting + static void initFSArgumentHandler(FSConfigToCSConfigConverter converter) { + ResourceManager.fsConfigConversionArgumentHandler = + new FSConfigToCSConfigArgumentHandler(converter); + } + + private static FSConfigToCSConfigConverter initFSConfigConverter() { + FSConfigToCSConfigRuleHandler ruleHandler = + new FSConfigToCSConfigRuleHandler(); + return new FSConfigToCSConfigConverter(ruleHandler); + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index fe7379a5ab..d7426833ac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -96,6 +96,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerPreemptEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ReleaseContainerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; import org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerContext; import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey; import org.apache.hadoop.yarn.server.utils.BuilderUtils; @@ -182,6 +183,8 @@ public abstract class AbstractYarnScheduler protected SchedulingMonitorManager schedulingMonitorManager = new SchedulingMonitorManager(); + private boolean migration; + /** * Construct the service. * @@ -197,6 +200,9 @@ public AbstractYarnScheduler(String name) { @Override public void serviceInit(Configuration conf) throws Exception { + migration = + conf.getBoolean(FairSchedulerConfiguration.MIGRATION_MODE, false); + nmExpireInterval = conf.getInt(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, YarnConfiguration.DEFAULT_RM_NM_EXPIRY_INTERVAL_MS); @@ -209,7 +215,10 @@ public void serviceInit(Configuration conf) throws Exception { nodeTracker.setConfiguredMaxAllocationWaitTime( configuredMaximumAllocationWaitTime); maxClusterLevelAppPriority = getMaxPriorityFromConf(conf); - this.releaseCache = new Timer("Pending Container Clear Timer"); + if (!migration) { + this.releaseCache = new Timer("Pending Container Clear Timer"); + } + autoUpdateContainers = conf.getBoolean(YarnConfiguration.RM_AUTO_UPDATE_CONTAINERS, YarnConfiguration.DEFAULT_RM_AUTO_UPDATE_CONTAINERS); @@ -227,11 +236,14 @@ public void serviceInit(Configuration conf) throws Exception { @Override protected void serviceStart() throws Exception { - if (updateThread != null) { - updateThread.start(); + if (!migration) { + if (updateThread != null) { + updateThread.start(); + } + schedulingMonitorManager.startAll(); + createReleaseCache(); } - schedulingMonitorManager.startAll(); - createReleaseCache(); + super.serviceStart(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java index 7f06521102..8b3567ffdf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java @@ -243,18 +243,34 @@ public int getUserMaxApps(String user) { return (maxApps == null) ? userMaxAppsDefault : maxApps; } + public Map getUserMaxApps() { + return userMaxApps; + } + @VisibleForTesting int getQueueMaxApps(String queue) { Integer maxApps = queueMaxApps.get(queue); return (maxApps == null) ? queueMaxAppsDefault : maxApps; } + public int getQueueMaxAppsDefault() { + return queueMaxAppsDefault; + } + + public int getUserMaxAppsDefault() { + return userMaxAppsDefault; + } + @VisibleForTesting float getQueueMaxAMShare(String queue) { Float maxAMShare = queueMaxAMShares.get(queue); return (maxAMShare == null) ? queueMaxAMShareDefault : maxAMShare; } + public float getQueueMaxAMShareDefault() { + return queueMaxAMShareDefault; + } + /** * Get the minimum resource allocation for the given queue. * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java index 44049fdf59..1ba7bf2972 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java @@ -137,4 +137,9 @@ void setPercentage(String name, double value) { } } } + + public double[] getPercentages() { + return percentages == null ? null : + Arrays.copyOf(percentages, percentages.length); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java index c22fdb05c2..cca0875618 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java @@ -184,6 +184,10 @@ public Resource getMaxShare() { return result; } + public ConfigurableResource getRawMaxShare() { + return maxShare; + } + public Resource getReservedResource() { reservedResource.setMemorySize(metrics.getReservedMB()); reservedResource.setVirtualCores(metrics.getReservedVirtualCores()); @@ -207,7 +211,7 @@ public int getMaxRunningApps() { } @VisibleForTesting - protected float getMaxAMShare() { + public float getMaxAMShare() { return maxAMShare; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 151a7ab086..04bbe0ff68 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -213,6 +213,8 @@ public class FairScheduler extends @VisibleForTesting Resource reservationThreshold; + private boolean migration; + public FairScheduler() { super(FairScheduler.class.getName()); context = new FSContext(this); @@ -1428,7 +1430,7 @@ private void initScheduler(Configuration conf) throws IOException { allocConf = new AllocationConfiguration(this); queueMgr.initialize(); - if (continuousSchedulingEnabled) { + if (continuousSchedulingEnabled && !migration) { // Continuous scheduling is deprecated log it on startup LOG.warn("Continuous scheduling is turned ON. It is deprecated " + "because it can cause scheduler slowness due to locking issues. " + @@ -1441,7 +1443,7 @@ private void initScheduler(Configuration conf) throws IOException { schedulingThread.setDaemon(true); } - if (this.conf.getPreemptionEnabled()) { + if (this.conf.getPreemptionEnabled() && !migration) { createPreemptionThread(); } } finally { @@ -1495,11 +1497,15 @@ private void startSchedulerThreads() { @Override public void serviceInit(Configuration conf) throws Exception { + migration = + conf.getBoolean(FairSchedulerConfiguration.MIGRATION_MODE, false); initScheduler(conf); super.serviceInit(conf); - // Initialize SchedulingMonitorManager - schedulingMonitorManager.initialize(rmContext, conf); + if (!migration) { + // Initialize SchedulingMonitorManager + schedulingMonitorManager.initialize(rmContext, conf); + } } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java index cfe07c9ef4..f83a492962 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java @@ -85,29 +85,39 @@ public class FairSchedulerConfiguration extends Configuration { private static final String CONF_PREFIX = "yarn.scheduler.fair."; + /** + * Used during FS->CS conversion. When enabled, background threads are + * not started. This property should NOT be used by end-users! + */ + public static final String MIGRATION_MODE = CONF_PREFIX + "migration.mode"; + public static final String ALLOCATION_FILE = CONF_PREFIX + "allocation.file"; protected static final String DEFAULT_ALLOCATION_FILE = "fair-scheduler.xml"; /** Whether pools can be created that were not specified in the FS configuration file */ - protected static final String ALLOW_UNDECLARED_POOLS = CONF_PREFIX + "allow-undeclared-pools"; - protected static final boolean DEFAULT_ALLOW_UNDECLARED_POOLS = true; + public static final String ALLOW_UNDECLARED_POOLS = CONF_PREFIX + + "allow-undeclared-pools"; + public static final boolean DEFAULT_ALLOW_UNDECLARED_POOLS = true; /** Whether to use the user name as the queue name (instead of "default") if * the request does not specify a queue. */ - protected static final String USER_AS_DEFAULT_QUEUE = CONF_PREFIX + "user-as-default-queue"; - protected static final boolean DEFAULT_USER_AS_DEFAULT_QUEUE = true; + public static final String USER_AS_DEFAULT_QUEUE = CONF_PREFIX + + "user-as-default-queue"; + public static final boolean DEFAULT_USER_AS_DEFAULT_QUEUE = true; protected static final float DEFAULT_LOCALITY_THRESHOLD = -1.0f; /** Cluster threshold for node locality. */ - protected static final String LOCALITY_THRESHOLD_NODE = CONF_PREFIX + "locality.threshold.node"; - protected static final float DEFAULT_LOCALITY_THRESHOLD_NODE = + public static final String LOCALITY_THRESHOLD_NODE = CONF_PREFIX + + "locality.threshold.node"; + public static final float DEFAULT_LOCALITY_THRESHOLD_NODE = DEFAULT_LOCALITY_THRESHOLD; /** Cluster threshold for rack locality. */ - protected static final String LOCALITY_THRESHOLD_RACK = CONF_PREFIX + "locality.threshold.rack"; - protected static final float DEFAULT_LOCALITY_THRESHOLD_RACK = + public static final String LOCALITY_THRESHOLD_RACK = CONF_PREFIX + + "locality.threshold.rack"; + public static final float DEFAULT_LOCALITY_THRESHOLD_RACK = DEFAULT_LOCALITY_THRESHOLD; /** @@ -139,10 +149,10 @@ public class FairSchedulerConfiguration extends Configuration { * {@link #ASSIGN_MULTIPLE} to improve container allocation ramp up. */ @Deprecated - protected static final String CONTINUOUS_SCHEDULING_ENABLED = CONF_PREFIX + + public static final String CONTINUOUS_SCHEDULING_ENABLED = CONF_PREFIX + "continuous-scheduling-enabled"; @Deprecated - protected static final boolean DEFAULT_CONTINUOUS_SCHEDULING_ENABLED = false; + public static final boolean DEFAULT_CONTINUOUS_SCHEDULING_ENABLED = false; /** * Sleep time of each pass in continuous scheduling (5ms in default). @@ -150,21 +160,22 @@ public class FairSchedulerConfiguration extends Configuration { * Only used when {@link #CONTINUOUS_SCHEDULING_ENABLED} is enabled */ @Deprecated - protected static final String CONTINUOUS_SCHEDULING_SLEEP_MS = CONF_PREFIX + + public static final String CONTINUOUS_SCHEDULING_SLEEP_MS = CONF_PREFIX + "continuous-scheduling-sleep-ms"; @Deprecated - protected static final int DEFAULT_CONTINUOUS_SCHEDULING_SLEEP_MS = 5; + public static final int DEFAULT_CONTINUOUS_SCHEDULING_SLEEP_MS = 5; /** Whether preemption is enabled. */ - protected static final String PREEMPTION = CONF_PREFIX + "preemption"; - protected static final boolean DEFAULT_PREEMPTION = false; + public static final String PREEMPTION = CONF_PREFIX + "preemption"; + public static final boolean DEFAULT_PREEMPTION = false; protected static final String PREEMPTION_THRESHOLD = CONF_PREFIX + "preemption.cluster-utilization-threshold"; protected static final float DEFAULT_PREEMPTION_THRESHOLD = 0.8f; - protected static final String WAIT_TIME_BEFORE_KILL = CONF_PREFIX + "waitTimeBeforeKill"; - protected static final int DEFAULT_WAIT_TIME_BEFORE_KILL = 15000; + public static final String WAIT_TIME_BEFORE_KILL = CONF_PREFIX + + "waitTimeBeforeKill"; + public static final int DEFAULT_WAIT_TIME_BEFORE_KILL = 15000; /** * Postfix for resource allocation increments in the @@ -181,18 +192,19 @@ public class FairSchedulerConfiguration extends Configuration { * This is intended to be a backdoor on production clusters, and hence * intentionally not documented. */ - protected static final String WAIT_TIME_BEFORE_NEXT_STARVATION_CHECK_MS = + public static final String WAIT_TIME_BEFORE_NEXT_STARVATION_CHECK_MS = CONF_PREFIX + "waitTimeBeforeNextStarvationCheck"; - protected static final long + public static final long DEFAULT_WAIT_TIME_BEFORE_NEXT_STARVATION_CHECK_MS = 10000; /** Whether to assign multiple containers in one check-in. */ public static final String ASSIGN_MULTIPLE = CONF_PREFIX + "assignmultiple"; - protected static final boolean DEFAULT_ASSIGN_MULTIPLE = false; + public static final boolean DEFAULT_ASSIGN_MULTIPLE = false; /** Whether to give more weight to apps requiring many resources. */ - protected static final String SIZE_BASED_WEIGHT = CONF_PREFIX + "sizebasedweight"; - protected static final boolean DEFAULT_SIZE_BASED_WEIGHT = false; + public static final String SIZE_BASED_WEIGHT = CONF_PREFIX + + "sizebasedweight"; + public static final boolean DEFAULT_SIZE_BASED_WEIGHT = false; /** Maximum number of containers to assign on each check-in. */ public static final String DYNAMIC_MAX_ASSIGN = @@ -203,8 +215,8 @@ public class FairSchedulerConfiguration extends Configuration { * Specify exact number of containers to assign on each heartbeat, if dynamic * max assign is turned off. */ - protected static final String MAX_ASSIGN = CONF_PREFIX + "max.assign"; - protected static final int DEFAULT_MAX_ASSIGN = -1; + public static final String MAX_ASSIGN = CONF_PREFIX + "max.assign"; + public static final int DEFAULT_MAX_ASSIGN = -1; /** The update interval for calculating resources in FairScheduler .*/ public static final String UPDATE_INTERVAL_MS = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/ConversionException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/ConversionException.java new file mode 100644 index 0000000000..bb228596b7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/ConversionException.java @@ -0,0 +1,37 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; + +/** + * Thrown when the FS-to-CS converter logic encounters a + * condition from which it cannot recover (eg. unsupported + * settings). + * + */ +public class ConversionException extends RuntimeException { + private static final long serialVersionUID = 4161836727287317835L; + + public ConversionException(String message) { + super(message); + } + + public ConversionException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java new file mode 100644 index 0000000000..89ce3cb39e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.MissingArgumentException; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; + +/** + * Parses arguments passed to the FS->CS converter. + * If the arguments are valid, it calls the converter itself. + * + */ +public class FSConfigToCSConfigArgumentHandler { + private static final Logger LOG = + LoggerFactory.getLogger(FSConfigToCSConfigArgumentHandler.class); + private final FSConfigToCSConfigConverter converter; + + public FSConfigToCSConfigArgumentHandler(FSConfigToCSConfigConverter + converter) { + this.converter = converter; + } + + /** + * Represents options for the converter CLI. + * + */ + public enum CliOption { + YARN_SITE("yarn-site.xml", "y", "yarnsiteconfig", + "Path to a valid yarn-site.xml config file", true, true), + + // fair-scheduler.xml is not mandatory + // if FairSchedulerConfiguration.ALLOCATION_FILE is defined in yarn-site.xml + FAIR_SCHEDULER("fair-scheduler.xml", "f", "fsconfig", + "Path to a valid fair-scheduler.xml config file", false, true), + CONVERSION_RULES("conversion rules config file", "r", "rulesconfig", + "Optional parameter. If given, should specify a valid path to the " + + "conversion rules file (property format).", false, true), + CONSOLE_MODE("console mode", "p", "print", + "If defined, the converted configuration will " + + "only be emitted to the console.", false, false), + CLUSTER_RESOURCE("cluster resource", "c", "cluster-resource", + "Needs to be given if maxResources is defined as percentages " + + "for any queue, otherwise this parameter can be omitted.", + false, true), + OUTPUT_DIR("output directory", "o", "output-directory", + "Output directory for yarn-site.xml and" + + " capacity-scheduler.xml files." + + "Must have write permission for user who is running this script.", + true, true); + + private final String name; + private final String shortSwitch; + private final String longSwitch; + private final String description; + private final boolean required; + private final boolean hasArg; + + CliOption(String name, String shortSwitch, String longSwitch, + String description, boolean required, boolean hasArg) { + this.name = name; + this.shortSwitch = shortSwitch; + this.longSwitch = longSwitch; + this.description = description; + this.required = required; + this.hasArg = hasArg; + } + + public Option createCommonsCliOption() { + Option option = new Option(shortSwitch, longSwitch, hasArg, description); + option.setRequired(required); + return option; + } + + public String getAsArgumentString() { + return shortSwitch + "|" + longSwitch + ": " + description; + } + } + + public int parseAndConvert(String[] args) throws Exception { + Options opts = createOptions(); + + try { + CommandLine cliParser = new GnuParser().parse(opts, args); + checkOptionPresent(cliParser, CliOption.YARN_SITE); + checkOptionPresent(cliParser, CliOption.OUTPUT_DIR); + + FSConfigToCSConfigConverterParams params = validateInputFiles(cliParser); + converter.convert(params); + } catch (MissingArgumentException e) { + String msg = "Missing argument for options" + e.getMessage(); + logAndStdErr(e, msg); + return -1; + } catch (PreconditionException e) { + String msg = "Cannot start FS config conversion due to the following" + + " precondition error: " + e.getMessage(); + logAndStdErr(e, msg); + return -1; + } catch (UnsupportedPropertyException e) { + String msg = "Unsupported property/setting encountered during FS config " + + "conversion: " + e.getMessage(); + logAndStdErr(e, msg); + return -1; + } catch (ConversionException | IllegalArgumentException e) { + String msg = "Fatal error during FS config conversion: " + e.getMessage(); + logAndStdErr(e, msg); + return -1; + } + + return 0; + } + + private void logAndStdErr(Exception e, String msg) { + LOG.error(msg, e); + System.err.println(msg); + } + + private Options createOptions() { + Options opts = new Options(); + + for (CliOption cliOption : CliOption.values()) { + opts.addOption(cliOption.createCommonsCliOption()); + } + + return opts; + } + + private FSConfigToCSConfigConverterParams validateInputFiles( + CommandLine cliParser) { + String yarnSiteXmlFile = + cliParser.getOptionValue(CliOption.YARN_SITE.shortSwitch); + String fairSchedulerXmlFile = + cliParser.getOptionValue(CliOption.FAIR_SCHEDULER.shortSwitch); + String conversionRulesFile = + cliParser.getOptionValue(CliOption.CONVERSION_RULES.shortSwitch); + String outputDir = + cliParser.getOptionValue(CliOption.OUTPUT_DIR.shortSwitch); + + checkFile(CliOption.YARN_SITE, yarnSiteXmlFile); + checkFile(CliOption.FAIR_SCHEDULER, fairSchedulerXmlFile); + checkFile(CliOption.CONVERSION_RULES, conversionRulesFile); + checkDirectory(CliOption.OUTPUT_DIR, outputDir); + + return FSConfigToCSConfigConverterParams.Builder.create() + .withYarnSiteXmlConfig(yarnSiteXmlFile) + .withFairSchedulerXmlConfig(fairSchedulerXmlFile) + .withConversionRulesConfig(conversionRulesFile) + .withClusterResource( + cliParser.getOptionValue(CliOption.CLUSTER_RESOURCE.shortSwitch)) + .withConsole(cliParser.hasOption(CliOption.CONSOLE_MODE.shortSwitch)) + .withOutputDirectory(outputDir) + .build(); + } + + private static void checkOptionPresent(CommandLine cliParser, + CliOption cliOption) { + if (!cliParser.hasOption(cliOption.shortSwitch)) { + throw new PreconditionException( + String.format("Missing %s parameter " + "(switch: %s|%s).", + cliOption.name, cliOption.shortSwitch, cliOption.longSwitch)); + } + } + + private static void checkFile(CliOption cliOption, String filePath) { + checkFileInternal(cliOption, filePath, true); + } + + private static void checkDirectory(CliOption cliOption, String dirPath) { + checkFileInternal(cliOption, dirPath, false); + } + + private static void checkFileInternal(CliOption cliOption, String filePath, + boolean isFile) { + //We can safely ignore null here as files / dirs were checked before + if (filePath == null) { + return; + } + + File file = new File(filePath); + if (isFile && file.isDirectory()) { + throw new PreconditionException( + String.format("Specified path %s is a directory but should be " + + " a file (As value of parameter %s)", filePath, cliOption.name)); + } else if (!isFile && !file.isDirectory()) { + throw new PreconditionException( + String.format("Specified path %s is not a directory " + + "(As value of parameter %s)", filePath, cliOption.name)); + } else if (!file.exists()) { + throw new PreconditionException( + String.format("Specified path %s does not exist " + + "(As value of parameter %s)", filePath, cliOption.name)); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java new file mode 100644 index 0000000000..469e050f25 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java @@ -0,0 +1,368 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; + +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.authorize.AccessControlList; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.security.AccessType; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; +import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfigurationException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.ConfigurableResource; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSParentQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; +import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.annotations.VisibleForTesting; + +/** + * Converts Fair Scheduler configuration (site and fair-scheduler.xml) + * to Capacity Scheduler. The mapping is not 100% perfect due to + * feature gaps. These will be addressed in the future. + */ +public class FSConfigToCSConfigConverter { + public static final Logger LOG = LoggerFactory.getLogger( + FSConfigToCSConfigConverter.class.getName()); + private static final String YARN_SITE_XML = "yarn-site.xml"; + private static final String CAPACITY_SCHEDULER_XML = + "capacity-scheduler.xml"; + private static final String FAIR_SCHEDULER_XML = + "fair-scheduler.xml"; + + public static final String WARNING_TEXT = + "WARNING: This feature is experimental and not intended " + + "for production use!"; + + + private Resource clusterResource; + private boolean preemptionEnabled = false; + private int queueMaxAppsDefault; + private float queueMaxAMShareDefault; + private boolean autoCreateChildQueues = false; + private boolean sizeBasedWeight = false; + private boolean userAsDefaultQueue = false; + + private Configuration yarnSiteConfig; + private Configuration capacitySchedulerConfig; + private FSConfigToCSConfigRuleHandler ruleHandler; + + private OutputStream yarnSiteOutputStream; + private OutputStream capacitySchedulerOutputStream; + private boolean consoleMode = false; + + public FSConfigToCSConfigConverter(FSConfigToCSConfigRuleHandler + ruleHandler) { + this.ruleHandler = ruleHandler; + this.yarnSiteOutputStream = System.out; + this.capacitySchedulerOutputStream = System.out; + } + + public void convert(FSConfigToCSConfigConverterParams params) + throws Exception { + validateParams(params); + prepareOutputFiles(params.getOutputDirectory(), params.isConsole()); + loadConversionRules(params.getConversionRulesConfig()); + Configuration conf = createConfiguration(params); + handleFairSchedulerConfig(params, conf); + + this.clusterResource = getClusterResource(params); + convert(conf); + } + + private void prepareOutputFiles(String outputDirectory, boolean console) + throws FileNotFoundException { + if (console) { + LOG.info("Console mode is enabled, " + YARN_SITE_XML + " and" + + " " + CAPACITY_SCHEDULER_XML + " will be only emitted " + + "to the console!"); + this.consoleMode = true; + return; + } + File yarnSiteXmlOutput = new File(outputDirectory, + YARN_SITE_XML); + File schedulerXmlOutput = new File(outputDirectory, + CAPACITY_SCHEDULER_XML); + LOG.info("Output directory for " + YARN_SITE_XML + " and" + + " " + CAPACITY_SCHEDULER_XML + " is: {}", outputDirectory); + + this.yarnSiteOutputStream = new FileOutputStream(yarnSiteXmlOutput); + this.capacitySchedulerOutputStream = + new FileOutputStream(schedulerXmlOutput); + } + + private void validateParams(FSConfigToCSConfigConverterParams params) { + if (params.getYarnSiteXmlConfig() == null) { + throw new PreconditionException("" + YARN_SITE_XML + " configuration " + + "is not defined but it is mandatory!"); + } else if (params.getOutputDirectory() == null && !params.isConsole()) { + throw new PreconditionException("Output directory configuration " + + "is not defined but it is mandatory!"); + } + } + + private Resource getClusterResource( + FSConfigToCSConfigConverterParams params) { + Resource resource = null; + if (params.getClusterResource() != null) { + ConfigurableResource configurableResource; + try { + configurableResource = FairSchedulerConfiguration + .parseResourceConfigValue(params.getClusterResource()); + } catch (AllocationConfigurationException e) { + throw new ConversionException("Error while parsing resource.", e); + } + resource = configurableResource.getResource(); + } + return resource; + } + + private void loadConversionRules(String rulesFile) throws IOException { + if (rulesFile != null) { + LOG.info("Reading conversion rules file from: " + rulesFile); + this.ruleHandler.loadRulesFromFile(rulesFile); + } else { + LOG.info("Conversion rules file is not defined, " + + "using default conversion config!"); + } + } + + private Configuration createConfiguration( + FSConfigToCSConfigConverterParams params) { + Configuration conf = new YarnConfiguration(); + conf.addResource(new Path(params.getYarnSiteXmlConfig())); + conf.setBoolean(FairSchedulerConfiguration.MIGRATION_MODE, true); + return conf; + } + + private void handleFairSchedulerConfig( + FSConfigToCSConfigConverterParams params, Configuration conf) { + String fairSchedulerXmlConfig = params.getFairSchedulerXmlConfig(); + + // Don't override allocation file in conf yet, as it would ruin the second + // condition here + if (fairSchedulerXmlConfig != null) { + LOG.info("Using explicitly defined " + FAIR_SCHEDULER_XML); + } else if (conf.get(FairSchedulerConfiguration.ALLOCATION_FILE) != null) { + LOG.info("Using " + FAIR_SCHEDULER_XML + " defined in " + + YARN_SITE_XML + " by key: " + + FairSchedulerConfiguration.ALLOCATION_FILE); + } else { + throw new PreconditionException("" + FAIR_SCHEDULER_XML + + " is not defined neither in " + YARN_SITE_XML + + "(with property: " + FairSchedulerConfiguration.ALLOCATION_FILE + + ") nor directly with its own parameter!"); + } + + // We can now safely override allocation file in conf + if (fairSchedulerXmlConfig != null) { + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, + params.getFairSchedulerXmlConfig()); + } + } + + @VisibleForTesting + void convert(Configuration conf) throws Exception { + System.out.println(WARNING_TEXT); + + // initialize Fair Scheduler + RMContext ctx = new RMContextImpl(); + PlacementManager placementManager = new PlacementManager(); + ctx.setQueuePlacementManager(placementManager); + + FairScheduler fs = new FairScheduler(); + fs.setRMContext(ctx); + fs.init(conf); + + AllocationConfiguration allocConf = fs.getAllocationConfiguration(); + queueMaxAppsDefault = allocConf.getQueueMaxAppsDefault(); + queueMaxAMShareDefault = allocConf.getQueueMaxAMShareDefault(); + + yarnSiteConfig = new Configuration(false); + capacitySchedulerConfig = new Configuration(false); + + checkUserMaxApps(allocConf); + checkUserMaxAppsDefault(allocConf); + + convertYarnSiteXml(conf); + convertCapacitySchedulerXml(fs); + + if (consoleMode) { + System.out.println("======= " + CAPACITY_SCHEDULER_XML + " ======="); + } + capacitySchedulerConfig.writeXml(capacitySchedulerOutputStream); + + if (consoleMode) { + System.out.println(); + System.out.println("======= " + YARN_SITE_XML + " ======="); + } + yarnSiteConfig.writeXml(yarnSiteOutputStream); + } + + @VisibleForTesting + void setYarnSiteOutputStream(OutputStream out) { + this.yarnSiteOutputStream = out; + } + + @VisibleForTesting + void setCapacitySchedulerConfigOutputStream(OutputStream out) { + this.capacitySchedulerOutputStream = out; + } + + private void convertYarnSiteXml(Configuration conf) { + FSYarnSiteConverter siteConverter = + new FSYarnSiteConverter(); + siteConverter.convertSiteProperties(conf, yarnSiteConfig); + + autoCreateChildQueues = siteConverter.isAutoCreateChildQueues(); + preemptionEnabled = siteConverter.isPreemptionEnabled(); + sizeBasedWeight = siteConverter.isSizeBasedWeight(); + userAsDefaultQueue = siteConverter.isUserAsDefaultQueue(); + + checkReservationSystem(conf); + } + + private void convertCapacitySchedulerXml(FairScheduler fs) { + FSParentQueue rootQueue = fs.getQueueManager().getRootQueue(); + emitDefaultMaxApplications(); + emitDefaultMaxAMShare(); + FSQueueConverter queueConverter = new FSQueueConverter(ruleHandler, + capacitySchedulerConfig, + preemptionEnabled, + sizeBasedWeight, + autoCreateChildQueues, + clusterResource, + queueMaxAMShareDefault, + queueMaxAppsDefault); + queueConverter.convertQueueHierarchy(rootQueue); + emitACLs(fs); + + PlacementManager placementManager = + fs.getRMContext().getQueuePlacementManager(); + + if (placementManager.getPlacementRules().size() > 0) { + QueuePlacementConverter placementConverter = + new QueuePlacementConverter(); + Map properties = + placementConverter.convertPlacementPolicy(placementManager, + ruleHandler, userAsDefaultQueue); + properties.forEach((k, v) -> capacitySchedulerConfig.set(k, v)); + } + + // Validate ordering policy + if (queueConverter.isDrfPolicyUsedOnQueueLevel()) { + if (queueConverter.isFifoOrFairSharePolicyUsed()) { + throw new ConversionException( + "DRF ordering policy cannot be used together with fifo/fair"); + } else { + capacitySchedulerConfig.set( + CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, + DominantResourceCalculator.class.getCanonicalName()); + } + } + } + + private void emitDefaultMaxApplications() { + if (queueMaxAppsDefault != Integer.MAX_VALUE) { + capacitySchedulerConfig.set( + CapacitySchedulerConfiguration.MAXIMUM_SYSTEM_APPLICATIONS, + String.valueOf(queueMaxAppsDefault)); + } + } + + private void emitDefaultMaxAMShare() { + capacitySchedulerConfig.set( + CapacitySchedulerConfiguration. + MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, + String.valueOf(queueMaxAMShareDefault)); + } + + private void emitACLs(FairScheduler fs) { + fs.getAllocationConfiguration().getQueueAcls() + .forEach(this::generateQueueAcl); + } + + private void generateQueueAcl(String queue, + Map access) { + AccessControlList submitAcls = access.get(AccessType.SUBMIT_APP); + AccessControlList adminAcls = access.get(AccessType.ADMINISTER_QUEUE); + + if (!submitAcls.getGroups().isEmpty() || + !submitAcls.getUsers().isEmpty()) { + capacitySchedulerConfig.set(PREFIX + queue + ".acl_submit_applications", + submitAcls.getAclString()); + } + + if (!adminAcls.getGroups().isEmpty() || + !adminAcls.getUsers().isEmpty()) { + capacitySchedulerConfig.set(PREFIX + queue + ".acl_administer_queue", + adminAcls.getAclString()); + } + } + + private void checkReservationSystem(Configuration conf) { + if (conf.getBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE, + YarnConfiguration.DEFAULT_RM_RESERVATION_SYSTEM_ENABLE)) { + ruleHandler.handleReservationSystem(); + } + } + + private void checkUserMaxApps(AllocationConfiguration allocConf) { + if (allocConf.getUserMaxApps() != null + && allocConf.getUserMaxApps().size() > 0) { + ruleHandler.handleUserMaxApps(); + } + } + + private void checkUserMaxAppsDefault(AllocationConfiguration allocConf) { + if (allocConf.getUserMaxAppsDefault() > 0) { + ruleHandler.handleUserMaxAppsDefault(); + } + } + + @VisibleForTesting + Resource getClusterResource() { + return clusterResource; + } + + @VisibleForTesting + public void setClusterResource(Resource clusterResource) { + this.clusterResource = clusterResource; + } + + @VisibleForTesting + FSConfigToCSConfigRuleHandler getRuleHandler() { + return ruleHandler; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverterParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverterParams.java new file mode 100644 index 0000000000..b083b252d4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverterParams.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; + +/** + * POJO that holds values for the FS->CS converter. + * + */ +public final class FSConfigToCSConfigConverterParams { + private String yarnSiteXmlConfig; + private String fairSchedulerXmlConfig; + private String conversionRulesConfig; + private boolean console; + private String clusterResource; + private String outputDirectory; + + private FSConfigToCSConfigConverterParams() { + //must use builder + } + + public String getFairSchedulerXmlConfig() { + return fairSchedulerXmlConfig; + } + + public String getYarnSiteXmlConfig() { + return yarnSiteXmlConfig; + } + + public String getConversionRulesConfig() { + return conversionRulesConfig; + } + + public String getClusterResource() { + return clusterResource; + } + + public boolean isConsole() { + return console; + } + + public String getOutputDirectory() { + return outputDirectory; + } + + @Override + public String toString() { + return "FSConfigToCSConfigConverterParams{" + + "yarnSiteXmlConfig='" + yarnSiteXmlConfig + '\'' + + ", fairSchedulerXmlConfig='" + fairSchedulerXmlConfig + '\'' + + ", conversionRulesConfig='" + conversionRulesConfig + '\'' + + ", clusterResource='" + clusterResource + '\'' + + ", console=" + console + + '}'; + } + + /** + * Builder that can construct FSConfigToCSConfigConverterParams objects. + * + */ + public static final class Builder { + private String yarnSiteXmlConfig; + private String fairSchedulerXmlConfig; + private String conversionRulesConfig; + private boolean console; + private String clusterResource; + private String outputDirectory; + + private Builder() { + } + + public static Builder create() { + return new Builder(); + } + + public Builder withYarnSiteXmlConfig(String config) { + this.yarnSiteXmlConfig = config; + return this; + } + + public Builder withFairSchedulerXmlConfig(String config) { + this.fairSchedulerXmlConfig = config; + return this; + } + + public Builder withConversionRulesConfig(String config) { + this.conversionRulesConfig = config; + return this; + } + + public Builder withClusterResource(String res) { + this.clusterResource = res; + return this; + } + + public Builder withConsole(boolean console) { + this.console = console; + return this; + } + + public Builder withOutputDirectory(String outputDir) { + this.outputDirectory = outputDir; + return this; + } + + public FSConfigToCSConfigConverterParams build() { + FSConfigToCSConfigConverterParams params = + new FSConfigToCSConfigConverterParams(); + params.clusterResource = this.clusterResource; + params.console = this.console; + params.fairSchedulerXmlConfig = this.fairSchedulerXmlConfig; + params.yarnSiteXmlConfig = this.yarnSiteXmlConfig; + params.conversionRulesConfig = this.conversionRulesConfig; + params.outputDirectory = this.outputDirectory; + return params; + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigRuleHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigRuleHandler.java new file mode 100644 index 0000000000..3f4455cd0e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigRuleHandler.java @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; + +import static java.lang.String.format; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import org.apache.commons.lang3.StringUtils; + +import com.google.common.annotations.VisibleForTesting; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Class that determines what should happen if the FS->CS converter + * encounters a property that is currently not supported. + * + * Acceptable values are either "abort" or "warning". + */ +public class FSConfigToCSConfigRuleHandler { + private static final Logger LOG = + LoggerFactory.getLogger(FSConfigToCSConfigRuleHandler.class); + + + public static final String MAX_CHILD_QUEUE_LIMIT = + "maxChildQueue.limit"; + + public static final String MAX_CAPACITY_PERCENTAGE = + "maxCapacityPercentage.action"; + + public static final String MAX_CHILD_CAPACITY = + "maxChildCapacity.action"; + + public static final String USER_MAX_RUNNING_APPS = + "userMaxRunningApps.action"; + + public static final String USER_MAX_APPS_DEFAULT = + "userMaxAppsDefault.action"; + + public static final String DYNAMIC_MAX_ASSIGN = + "dynamicMaxAssign.action"; + + public static final String SPECIFIED_NOT_FIRST = + "specifiedNotFirstRule.action"; + + public static final String RESERVATION_SYSTEM = + "reservationSystem.action"; + + public static final String QUEUE_AUTO_CREATE = + "queueAutoCreate.action"; + + @VisibleForTesting + enum RuleAction { + WARNING, + ABORT + } + + private Map actions; + private Properties properties; + + void loadRulesFromFile(String ruleFile) throws IOException { + if (ruleFile == null) { + throw new IllegalArgumentException("Rule file cannot be null!"); + } + + properties = new Properties(); + try (InputStream is = new FileInputStream(new File(ruleFile))) { + properties.load(is); + } + actions = new HashMap<>(); + initPropertyActions(); + } + + public FSConfigToCSConfigRuleHandler() { + properties = new Properties(); + actions = new HashMap<>(); + } + + @VisibleForTesting + FSConfigToCSConfigRuleHandler(Properties props) { + properties = props; + actions = new HashMap<>(); + initPropertyActions(); + } + + private void initPropertyActions() { + setActionForProperty(MAX_CAPACITY_PERCENTAGE); + setActionForProperty(MAX_CHILD_CAPACITY); + setActionForProperty(USER_MAX_RUNNING_APPS); + setActionForProperty(USER_MAX_APPS_DEFAULT); + setActionForProperty(DYNAMIC_MAX_ASSIGN); + setActionForProperty(SPECIFIED_NOT_FIRST); + setActionForProperty(RESERVATION_SYSTEM); + setActionForProperty(QUEUE_AUTO_CREATE); + } + + public void handleMaxCapacityPercentage(String queueName) { + handle(MAX_CAPACITY_PERCENTAGE, null, + format(" defined in percentages for queue %s", + queueName)); + } + + public void handleMaxChildCapacity() { + handle(MAX_CHILD_CAPACITY, "", null); + } + + public void handleChildQueueCount(String queue, int count) { + String value = properties.getProperty(MAX_CHILD_QUEUE_LIMIT); + if (value != null) { + if (StringUtils.isNumeric(value)) { + int maxChildQueue = Integer.parseInt(value); + if (count > maxChildQueue) { + throw new ConversionException( + format("Queue %s has too many children: %d", queue, count)); + } + } else { + throw new ConversionException( + "Rule setting: maxChildQueue.limit is not an integer"); + } + } + } + + public void handleUserMaxApps() { + handle(USER_MAX_RUNNING_APPS, "", null); + } + + public void handleUserMaxAppsDefault() { + handle(USER_MAX_APPS_DEFAULT, "", null); + } + + public void handleDynamicMaxAssign() { + handle(DYNAMIC_MAX_ASSIGN, + FairSchedulerConfiguration.DYNAMIC_MAX_ASSIGN, null); + } + + public void handleSpecifiedNotFirstRule() { + handle(SPECIFIED_NOT_FIRST, + null, + "The tag is not the first placement rule, this cannot be" + + " converted properly"); + } + + public void handleReservationSystem() { + handle(RESERVATION_SYSTEM, + null, + "Conversion of reservation system is not supported"); + } + + public void handleQueueAutoCreate(String placementRule) { + handle(QUEUE_AUTO_CREATE, + null, + format( + "Placement rules: queue auto-create is not supported (type: %s)", + placementRule)); + } + + private void handle(String actionName, String fsSetting, String message) { + RuleAction action = actions.get(actionName); + + if (action != null) { + switch (action) { + case ABORT: + String exceptionMessage; + if (message != null) { + exceptionMessage = message; + } else { + exceptionMessage = format("Setting %s is not supported", fsSetting); + } + throw new UnsupportedPropertyException(exceptionMessage); + case WARNING: + if (message != null) { + LOG.warn(message); + } else { + LOG.warn("Setting {} is not supported, ignoring conversion", + fsSetting); + } + break; + default: + throw new IllegalArgumentException( + "Unknown action " + action); + } + } + } + + private void setActionForProperty(String property) { + String action = properties.getProperty(property); + + if (action == null) { + LOG.info("No rule set for {}, defaulting to WARNING", property); + actions.put(property, RuleAction.WARNING); + } else if (action.equalsIgnoreCase("warning")) { + actions.put(property, RuleAction.WARNING); + } else if (action.equalsIgnoreCase("abort")) { + actions.put(property, RuleAction.ABORT); + } else { + LOG.warn("Unknown action {} set for rule {}, defaulting to WARNING", + action, property); + actions.put(property, RuleAction.WARNING); + } + } + + @VisibleForTesting + public Map getActions() { + return actions; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java new file mode 100644 index 0000000000..2346bb9eeb --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java @@ -0,0 +1,493 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; + +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.mutable.MutableBoolean; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.ConfigurableResource; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; +import org.apache.hadoop.yarn.util.resource.Resources; + +/** + * Converts a Fair Schedule queue hierarchy to Capacity Scheduler + * configuration. + * + */ +public class FSQueueConverter { + private static final int MAX_RUNNING_APPS_UNSET = Integer.MIN_VALUE; + private final Set leafQueueNames; + private final FSConfigToCSConfigRuleHandler ruleHandler; + private Configuration capacitySchedulerConfig; + private final boolean preemptionEnabled; + private final boolean sizeBasedWeight; + private final Resource clusterResource; + private final float queueMaxAMShareDefault; + private final boolean autoCreateChildQueues; + private final int queueMaxAppsDefault; + + private boolean fifoOrFairSharePolicyUsed; + private boolean drfPolicyUsedOnQueueLevel; + + @SuppressWarnings("checkstyle:parameternumber") + public FSQueueConverter(FSConfigToCSConfigRuleHandler ruleHandler, + Configuration capacitySchedulerConfig, + boolean preemptionEnabled, + boolean sizeBasedWeight, + boolean autoCreateChildQueues, + Resource clusterResource, + float queueMaxAMShareDefault, + int queueMaxAppsDefault) { + this.leafQueueNames = new HashSet<>(); + this.ruleHandler = ruleHandler; + this.capacitySchedulerConfig = capacitySchedulerConfig; + this.preemptionEnabled = preemptionEnabled; + this.sizeBasedWeight = sizeBasedWeight; + this.clusterResource = clusterResource; + this.queueMaxAMShareDefault = queueMaxAMShareDefault; + this.autoCreateChildQueues = autoCreateChildQueues; + this.queueMaxAppsDefault = queueMaxAppsDefault; + } + + @SuppressWarnings("checkstyle:linelength") + public void convertQueueHierarchy(FSQueue queue) { + List children = queue.getChildQueues(); + final String queueName = queue.getName(); + + if (queue instanceof FSLeafQueue) { + String shortName = getQueueShortName(queueName); + if (!leafQueueNames.add(shortName)) { + throw new ConversionException( + "Leaf queues must be unique, " + + shortName + " is defined at least twice"); + } + } + + emitChildQueues(queueName, children); + emitMaxAMShare(queueName, queue); + emitMaxRunningApps(queueName, queue); + emitMaxAllocations(queueName, queue); + emitPreemptionDisabled(queueName, queue); + + // TODO: COULD BE incorrect! Needs further clarifications + emitChildCapacity(queue); + emitMaximumCapacity(queueName, queue); + emitAutoCreateChildQueue(queueName); + emitSizeBasedWeight(queueName); + emitOrderingPolicy(queueName, queue); + checkMaxChildCapacitySetting(queue); + + for (FSQueue childQueue : children) { + convertQueueHierarchy(childQueue); + } + } + + public boolean isFifoOrFairSharePolicyUsed() { + return fifoOrFairSharePolicyUsed; + } + + public boolean isDrfPolicyUsedOnQueueLevel() { + return drfPolicyUsedOnQueueLevel; + } + + /** + * Generates yarn.scheduler.capacity.<queue-name>.queues. + * @param queueName + * @param children + */ + private void emitChildQueues(String queueName, List children) { + ruleHandler.handleChildQueueCount(queueName, children.size()); + + if (children.size() > 0) { + String childQueues = children.stream() + .map(child -> getQueueShortName(child.getName())) + .collect(Collectors.joining(",")); + + capacitySchedulerConfig.set(PREFIX + queueName + ".queues", childQueues); + } + } + + /** + * <maxAMShare> + * ==> yarn.scheduler.capacity.<queue-name>.maximum-am-resource-percent. + * @param queueName + * @param queue + */ + private void emitMaxAMShare(String queueName, FSQueue queue) { + float queueMaxAmShare = queue.getMaxAMShare(); + + // Direct floating point comparison is OK here + if (queueMaxAmShare != 0.0f + && queueMaxAmShare != queueMaxAMShareDefault + && queueMaxAmShare != -1.0f) { + capacitySchedulerConfig.set(PREFIX + queueName + + ".maximum-am-resource-percent", String.valueOf(queueMaxAmShare)); + } + + if (queueMaxAmShare == -1.0f) { + capacitySchedulerConfig.set(PREFIX + queueName + + ".maximum-am-resource-percent", "1.0"); + } + } + + /** + * <maxRunningApps> + * ==> yarn.scheduler.capacity.<queue-name>.maximum-applications. + * @param queueName + * @param queue + */ + private void emitMaxRunningApps(String queueName, FSQueue queue) { + if (queue.getMaxRunningApps() != MAX_RUNNING_APPS_UNSET + && queue.getMaxRunningApps() != queueMaxAppsDefault) { + capacitySchedulerConfig.set(PREFIX + queueName + ".maximum-applications", + String.valueOf(queue.getMaxRunningApps())); + } + } + + /** + * <maxResources> + * ==> yarn.scheduler.capacity.<queue-name>.maximum-capacity. + * @param queueName + * @param queue + */ + private void emitMaximumCapacity(String queueName, FSQueue queue) { + ConfigurableResource rawMaxShare = queue.getRawMaxShare(); + final Resource maxResource = rawMaxShare.getResource(); + + long memSize = 0; + long vCores = 0; + boolean defined = false; + + if (maxResource == null) { + if (rawMaxShare.getPercentages() != null) { + if (clusterResource == null) { + throw new ConversionException( + String.format(" defined in percentages for" + + " queue %s, but cluster resource parameter is not" + + " defined via CLI!", queueName)); + } + + ruleHandler.handleMaxCapacityPercentage(queueName); + + double[] percentages = rawMaxShare.getPercentages(); + int memIndex = ResourceUtils.getResourceTypeIndex().get("memory-mb"); + int vcoreIndex = ResourceUtils.getResourceTypeIndex().get("vcores"); + + memSize = (long) (percentages[memIndex] * + clusterResource.getMemorySize()); + vCores = (long) (percentages[vcoreIndex] * + clusterResource.getVirtualCores()); + defined = true; + } else { + throw new PreconditionException( + "Illegal ConfigurableResource = " + rawMaxShare); + } + } else if (isNotUnboundedResource(maxResource)) { + memSize = maxResource.getMemorySize(); + vCores = maxResource.getVirtualCores(); + defined = true; + } + + if (defined) { + capacitySchedulerConfig.set(PREFIX + queueName + ".maximum-capacity", + String.format("[memory=%d, vcores=%d]", memSize, vCores)); + } + } + + /** + * <maxContainerAllocation> + * ==> yarn.scheduler.capacity.<queue-name>.maximum-allocation-mb + * / vcores. + * @param queueName + * @param queue + */ + private void emitMaxAllocations(String queueName, FSQueue queue) { + Resource maxAllocation = queue.getMaximumContainerAllocation(); + + if (isNotUnboundedResource(maxAllocation)) { + long parentMaxVcores = Integer.MIN_VALUE; + long parentMaxMemory = Integer.MIN_VALUE; + + if (queue.getParent() != null) { + FSQueue parent = queue.getParent(); + Resource parentMaxAllocation = parent.getMaximumContainerAllocation(); + if (isNotUnboundedResource(parentMaxAllocation)) { + parentMaxVcores = parentMaxAllocation.getVirtualCores(); + parentMaxMemory = parentMaxAllocation.getMemorySize(); + } + } + + long maxVcores = maxAllocation.getVirtualCores(); + long maxMemory = maxAllocation.getMemorySize(); + + // only emit max allocation if it differs from the parent's setting + if (maxVcores != parentMaxVcores || maxMemory != parentMaxMemory) { + capacitySchedulerConfig.set(PREFIX + queueName + + ".maximum-allocation-mb", String.valueOf(maxMemory)); + + capacitySchedulerConfig.set(PREFIX + queueName + + ".maximum-allocation-vcores", String.valueOf(maxVcores)); + } + } + } + + /** + * <allowPreemptionFrom> + * ==> yarn.scheduler.capacity.<queue-name>.disable_preemption. + * @param queueName + * @param queue + */ + private void emitPreemptionDisabled(String queueName, FSQueue queue) { + if (preemptionEnabled && !queue.isPreemptable()) { + capacitySchedulerConfig.set(PREFIX + queueName + ".disable_preemption", + "true"); + } + } + + /** + * yarn.scheduler.fair.allow-undeclared-pools + * ==> yarn.scheduler.capacity.<queue-name> + * .auto-create-child-queue.enabled. + * @param queueName + */ + private void emitAutoCreateChildQueue(String queueName) { + if (autoCreateChildQueues) { + capacitySchedulerConfig.setBoolean(PREFIX + queueName + + ".auto-create-child-queue.enabled", true); + } + } + + /** + * yarn.scheduler.fair.sizebasedweight ==> + * yarn.scheduler.capacity.<queue-path> + * .ordering-policy.fair.enable-size-based-weight. + * @param queueName + */ + private void emitSizeBasedWeight(String queueName) { + if (sizeBasedWeight) { + capacitySchedulerConfig.setBoolean(PREFIX + queueName + + ".ordering-policy.fair.enable-size-based-weight", true); + } + } + + /** + * <schedulingPolicy> + * ==> yarn.scheduler.capacity.<queue-path>.ordering-policy. + * @param queueName + * @param queue + */ + private void emitOrderingPolicy(String queueName, FSQueue queue) { + String policy = queue.getPolicy().getName(); + + switch (policy) { + case FairSharePolicy.NAME: + capacitySchedulerConfig.set(PREFIX + queueName + + ".ordering-policy", FairSharePolicy.NAME); + fifoOrFairSharePolicyUsed = true; + break; + case FifoPolicy.NAME: + capacitySchedulerConfig.set(PREFIX + queueName + + ".ordering-policy", FifoPolicy.NAME); + fifoOrFairSharePolicyUsed = true; + break; + case DominantResourceFairnessPolicy.NAME: + // DRF is not supported on a queue level, + // it has to be global + drfPolicyUsedOnQueueLevel = true; + break; + default: + throw new ConversionException("Unexpected ordering policy " + + "on queue " + queueName + ": " + policy); + } + } + + /** + * weight + minResources + * ==> yarn.scheduler.capacity.<queue-name>.capacity. + * @param queue + */ + private void emitChildCapacity(FSQueue queue) { + List children = queue.getChildQueues(); + + int totalWeight = getTotalWeight(children); + Map capacities = getCapacities(totalWeight, children); + capacities + .forEach((key, value) -> capacitySchedulerConfig.set(PREFIX + key + + ".capacity", value.toString())); + } + + /** + * Missing feature, "leaf-queue-template.capacity" only accepts a single + * pct value. + * @param queue + */ + private void checkMaxChildCapacitySetting(FSQueue queue) { + if (queue.getMaxChildQueueResource() != null) { + Resource resource = queue.getMaxChildQueueResource().getResource(); + + if ((resource != null && isNotUnboundedResource(resource)) + || queue.getMaxChildQueueResource().getPercentages() != null) { + // Maximum child resource is defined + ruleHandler.handleMaxChildCapacity(); + } + } + } + + private Map getCapacities(int totalWeight, + List children) { + final BigDecimal hundred = new BigDecimal(100).setScale(3); + + if (children.size() == 0) { + return new HashMap<>(); + } else if (children.size() == 1) { + Map capacity = new HashMap<>(); + String queueName = children.get(0).getName(); + capacity.put(queueName, Capacity.newCapacity(hundred)); + + return capacity; + } else { + Map capacities = new HashMap<>(); + Map bdCapacities = new HashMap<>(); + + MutableBoolean needVerifySum = new MutableBoolean(true); + children + .stream() + .forEach(queue -> { + BigDecimal total = new BigDecimal(totalWeight); + BigDecimal weight = new BigDecimal(queue.getWeight()); + BigDecimal pct = weight + .setScale(5) + .divide(total, RoundingMode.HALF_UP) + .multiply(hundred) + .setScale(3); + + // defined? + if (Resources.none().compareTo(queue.getMinShare()) != 0) { + needVerifySum.setFalse(); + + /* TODO: Needs discussion. + * + * Probably it's not entirely correct this way! + * Eg. root.queue1 in FS translates to 33% + * capacity, but minResources is defined as 1vcore,8GB + * which is less than 33%. + * + * Therefore max(calculatedCapacity, minResource) is + * more sound. + */ + Resource minShare = queue.getMinShare(); + // TODO: in Phase-2, we have to deal with other resources as well + String capacity = String.format("[memory=%d,vcores=%d]", + minShare.getMemorySize(), minShare.getVirtualCores()); + capacities.put(queue.getName(), Capacity.newCapacity(capacity)); + } else { + capacities.put(queue.getName(), Capacity.newCapacity(pct)); + bdCapacities.put(queue.getName(), pct); + } + }); + + if (needVerifySum.isTrue()) { + BigDecimal totalPct = new BigDecimal(0); + for (Map.Entry entry : bdCapacities.entrySet()) { + totalPct = totalPct.add(entry.getValue()); + } + + // fix last value if total != 100.000 + if (!totalPct.equals(hundred)) { + BigDecimal tmp = new BigDecimal(0); + for (int i = 0; i < children.size() - 2; i++) { + tmp = tmp.add(bdCapacities.get(children.get(i).getQueueName())); + } + + String lastQueue = children.get(children.size() - 1).getName(); + BigDecimal corrected = hundred.subtract(tmp); + capacities.put(lastQueue, Capacity.newCapacity(corrected)); + } + } + + return capacities; + } + } + + private int getTotalWeight(List children) { + double sum = children + .stream() + .mapToDouble(c -> c.getWeight()) + .sum(); + return (int) sum; + } + + private String getQueueShortName(String queueName) { + int lastDot = queueName.lastIndexOf("."); + return queueName.substring(lastDot + 1); + } + + private boolean isNotUnboundedResource(Resource res) { + return Resources.unbounded().compareTo(res) != 0; + } + + /* + * Represents a queue capacity in either percentage + * or in absolute resources + */ + private static class Capacity { + private BigDecimal percentage; + private String absoluteResource; + + public static Capacity newCapacity(BigDecimal pct) { + Capacity capacity = new Capacity(); + capacity.percentage = pct; + capacity.absoluteResource = null; + + return capacity; + } + + public static Capacity newCapacity(String absoluteResource) { + Capacity capacity = new Capacity(); + capacity.percentage = null; + capacity.absoluteResource = absoluteResource; + + return capacity; + } + + @Override + public String toString() { + if (percentage != null) { + return percentage.toString(); + } else { + return absoluteResource; + } + } + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java new file mode 100644 index 0000000000..f6d023dbe9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSYarnSiteConverter.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; + +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; + +/** + * Converts a Fair Scheduler site configuration to Capacity Scheduler + * site configuration. + * + */ +public class FSYarnSiteConverter { + private boolean preemptionEnabled; + private boolean autoCreateChildQueues; + private boolean sizeBasedWeight; + private boolean userAsDefaultQueue; + + @SuppressWarnings({"deprecation", "checkstyle:linelength"}) + public void convertSiteProperties(Configuration conf, + Configuration yarnSiteConfig) { + yarnSiteConfig.set(YarnConfiguration.RM_SCHEDULER, + CapacityScheduler.class.getCanonicalName()); + + // TODO: deprecated property, check if necessary + if (conf.getBoolean( + FairSchedulerConfiguration.CONTINUOUS_SCHEDULING_ENABLED, + FairSchedulerConfiguration.DEFAULT_CONTINUOUS_SCHEDULING_ENABLED)) { + yarnSiteConfig.setBoolean( + CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE, true); + int interval = conf.getInt( + FairSchedulerConfiguration.CONTINUOUS_SCHEDULING_SLEEP_MS, + FairSchedulerConfiguration.DEFAULT_CONTINUOUS_SCHEDULING_SLEEP_MS); + yarnSiteConfig.setInt(PREFIX + + "schedule-asynchronously.scheduling-interval-ms", interval); + } + + String mbIncrementAllocation = + conf.get("yarn.resource-types.memory-mb.increment-allocation"); + if (mbIncrementAllocation != null) { + yarnSiteConfig.set("yarn.scheduler.minimum-allocation-mb", + mbIncrementAllocation); + } + + String vcoreIncrementAllocation = + conf.get("yarn.resource-types.vcores.increment-allocation"); + if (vcoreIncrementAllocation != null) { + yarnSiteConfig.set("yarn.scheduler.minimum-allocation-vcores", + vcoreIncrementAllocation); + } + + if (conf.getBoolean(FairSchedulerConfiguration.PREEMPTION, + FairSchedulerConfiguration.DEFAULT_PREEMPTION)) { + yarnSiteConfig.setBoolean( + YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true); + preemptionEnabled = true; + + int waitTimeBeforeKill = conf.getInt( + FairSchedulerConfiguration.WAIT_TIME_BEFORE_KILL, + FairSchedulerConfiguration.DEFAULT_WAIT_TIME_BEFORE_KILL); + yarnSiteConfig.setInt( + CapacitySchedulerConfiguration.PREEMPTION_WAIT_TIME_BEFORE_KILL, + waitTimeBeforeKill); + + long waitBeforeNextStarvationCheck = conf.getLong( + FairSchedulerConfiguration.WAIT_TIME_BEFORE_NEXT_STARVATION_CHECK_MS, + FairSchedulerConfiguration.DEFAULT_WAIT_TIME_BEFORE_NEXT_STARVATION_CHECK_MS); + yarnSiteConfig.setLong( + CapacitySchedulerConfiguration.PREEMPTION_MONITORING_INTERVAL, + waitBeforeNextStarvationCheck); + } + + if (conf.getBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, + FairSchedulerConfiguration.DEFAULT_ASSIGN_MULTIPLE)) { + yarnSiteConfig.setBoolean( + CapacitySchedulerConfiguration.ASSIGN_MULTIPLE_ENABLED, true); + } else { + yarnSiteConfig.setBoolean( + CapacitySchedulerConfiguration.ASSIGN_MULTIPLE_ENABLED, false); + } + + int maxAssign = conf.getInt(FairSchedulerConfiguration.MAX_ASSIGN, + FairSchedulerConfiguration.DEFAULT_MAX_ASSIGN); + if (maxAssign != FairSchedulerConfiguration.DEFAULT_MAX_ASSIGN) { + yarnSiteConfig.setInt( + CapacitySchedulerConfiguration.MAX_ASSIGN_PER_HEARTBEAT, + maxAssign); + } + + float localityThresholdNode = conf.getFloat( + FairSchedulerConfiguration.LOCALITY_THRESHOLD_NODE, + FairSchedulerConfiguration.DEFAULT_LOCALITY_THRESHOLD_NODE); + if (localityThresholdNode != + FairSchedulerConfiguration.DEFAULT_LOCALITY_THRESHOLD_NODE) { + yarnSiteConfig.setFloat(CapacitySchedulerConfiguration.NODE_LOCALITY_DELAY, + localityThresholdNode); + } + + float localityThresholdRack = conf.getFloat( + FairSchedulerConfiguration.LOCALITY_THRESHOLD_RACK, + FairSchedulerConfiguration.DEFAULT_LOCALITY_THRESHOLD_RACK); + if (localityThresholdRack != + FairSchedulerConfiguration.DEFAULT_LOCALITY_THRESHOLD_RACK) { + yarnSiteConfig.setFloat( + CapacitySchedulerConfiguration.RACK_LOCALITY_ADDITIONAL_DELAY, + localityThresholdRack); + } + + if (conf.getBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, + FairSchedulerConfiguration.DEFAULT_ALLOW_UNDECLARED_POOLS)) { + autoCreateChildQueues = true; + } + + if (conf.getBoolean(FairSchedulerConfiguration.SIZE_BASED_WEIGHT, + FairSchedulerConfiguration.DEFAULT_SIZE_BASED_WEIGHT)) { + sizeBasedWeight = true; + } + + if (conf.getBoolean(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, + FairSchedulerConfiguration.DEFAULT_USER_AS_DEFAULT_QUEUE)) { + userAsDefaultQueue = true; + } + } + + public boolean isPreemptionEnabled() { + return preemptionEnabled; + } + + public boolean isAutoCreateChildQueues() { + return autoCreateChildQueues; + } + + public boolean isSizeBasedWeight() { + return sizeBasedWeight; + } + + public boolean isUserAsDefaultQueue() { + return userAsDefaultQueue; + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/PreconditionException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/PreconditionException.java new file mode 100644 index 0000000000..3259ba2b67 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/PreconditionException.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; + +import org.apache.commons.cli.MissingArgumentException; + +/** + * Indicates that some preconditions were not met + * before FS->CS conversion. + * + */ +public class PreconditionException extends RuntimeException { + private static final long serialVersionUID = 7976747724949372164L; + + public PreconditionException(String message) { + super(message); + } + + public PreconditionException(String message, MissingArgumentException ex) { + super(message, ex); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java new file mode 100644 index 0000000000..2db808f2f1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.yarn.server.resourcemanager.placement.DefaultPlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.placement.FSPlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager; +import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.placement.PrimaryGroupPlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.placement.SecondaryGroupExistingPlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.placement.SpecifiedPlacementRule; +import org.apache.hadoop.yarn.server.resourcemanager.placement.UserPlacementRule; + +class QueuePlacementConverter { + + private static final String USER = "%user"; + private static final String PRIMARY_GROUP = "%primary_group"; + private static final String SECONDARY_GROUP = "%secondary_group"; + + Map convertPlacementPolicy(PlacementManager placementManager, + FSConfigToCSConfigRuleHandler ruleHandler, boolean userAsDefaultQueue) { + StringBuilder mapping = new StringBuilder(); + Map properties = new HashMap<>(); + + if (userAsDefaultQueue) { + mapping.append("u:" + USER + ":" + USER); + } + + int ruleCount = 0; + for (PlacementRule rule : placementManager.getPlacementRules()) { + if (((FSPlacementRule)rule).getCreateFlag()) { + ruleHandler.handleQueueAutoCreate(rule.getName()); + } + + ruleCount++; + if (rule instanceof UserPlacementRule) { + UserPlacementRule userRule = (UserPlacementRule) rule; + if (mapping.length() > 0) { + mapping.append(";"); + } + + // nested rule + if (userRule.getParentRule() != null) { + handleNestedRule(mapping, userRule); + } else { + if (!userAsDefaultQueue) { + mapping.append("u:" + USER + ":" + USER); + } + } + } else if (rule instanceof SpecifiedPlacementRule) { + if (ruleCount > 1) { + ruleHandler.handleSpecifiedNotFirstRule(); + } + properties.put( + "yarn.scheduler.capacity.queue-mappings-override.enable", "false"); + } else if (rule instanceof PrimaryGroupPlacementRule) { + if (mapping.length() > 0) { + mapping.append(";"); + } + mapping.append("u:" + USER + ":" + PRIMARY_GROUP); + } else if (rule instanceof DefaultPlacementRule) { + DefaultPlacementRule defaultRule = (DefaultPlacementRule) rule; + if (mapping.length() > 0) { + mapping.append(";"); + } + mapping.append("u:" + USER + ":").append(defaultRule.defaultQueueName); + } else if (rule instanceof SecondaryGroupExistingPlacementRule) { + // TODO: wait for YARN-9840 + mapping.append("u:" + USER + ":" + SECONDARY_GROUP); + } else { + throw new IllegalArgumentException("Unknown placement rule: " + rule); + } + } + + if (mapping.length() > 0) { + properties.put("yarn.scheduler.capacity.queue-mappings", + mapping.toString()); + } + + return properties; + } + + private void handleNestedRule(StringBuilder mapping, + UserPlacementRule userRule) { + PlacementRule pr = userRule.getParentRule(); + if (pr instanceof PrimaryGroupPlacementRule) { + // TODO: wait for YARN-9841 + mapping.append("u:" + USER + ":" + PRIMARY_GROUP + "." + USER); + } else if (pr instanceof SecondaryGroupExistingPlacementRule) { + // TODO: wait for YARN-9865 + mapping.append("u:" + USER + ":" + SECONDARY_GROUP + "." + USER); + } else if (pr instanceof DefaultPlacementRule) { + DefaultPlacementRule defaultRule = (DefaultPlacementRule) pr; + mapping.append("u:" + USER + ":") + .append(defaultRule.defaultQueueName) + .append("." + USER); + } else { + throw new UnsupportedOperationException("Unsupported nested rule: " + + pr.getClass().getCanonicalName()); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/UnsupportedPropertyException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/UnsupportedPropertyException.java new file mode 100644 index 0000000000..8f89966618 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/UnsupportedPropertyException.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; + +/** + * Thrown by the FS->CS converter if it encounters an + * unsupported property. + */ +public class UnsupportedPropertyException extends RuntimeException { + private static final long serialVersionUID = 5468104026818355871L; + + public UnsupportedPropertyException(String message) { + super(message); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/package-info.java new file mode 100644 index 0000000000..271b0cea31 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/package-info.java @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This package contains classes related to the Fair Scheduler -> + * Capacity Scheduler conversion. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java index f2073aeb55..a857c27f45 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java @@ -18,17 +18,8 @@ package org.apache.hadoop.yarn.server.resourcemanager; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.fail; - -import java.io.IOException; -import java.util.Collection; -import java.util.concurrent.TimeoutException; - -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.http.lib.StaticUserWebFilter; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.security.AuthenticationFilterInitializer; @@ -47,6 +38,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigConverter; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigConverterParams; import org.apache.hadoop.yarn.server.security.http.RMAuthenticationFilterInitializer; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; @@ -55,29 +49,51 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import org.mockito.ArgumentCaptor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.Collection; +import java.util.concurrent.TimeoutException; + +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.CONVERSION_RULES_FILE; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.FS_ALLOC_FILE; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.OUTPUT_DIR; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.YARN_SITE_XML; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.setupFSConfigConversionFiles; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; public class TestResourceManager { private static final Logger LOG = LoggerFactory.getLogger(TestResourceManager.class); - + private ResourceManager resourceManager = null; @Rule public ExpectedException thrown = ExpectedException.none(); + private FSConfigConverterTestCommons converterTestCommons; @Before public void setUp() throws Exception { - Configuration conf = new YarnConfiguration(); + YarnConfiguration conf = new YarnConfiguration(); UserGroupInformation.setConfiguration(conf); resourceManager = new ResourceManager(); resourceManager.init(conf); resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey(); resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey(); + + converterTestCommons = new FSConfigConverterTestCommons(); + converterTestCommons.setUp(); } @After public void tearDown() throws Exception { resourceManager.stop(); + converterTestCommons.tearDown(); } private org.apache.hadoop.yarn.server.resourcemanager.NodeManager @@ -357,4 +373,70 @@ public void testUserProvidedUGIConf() throws Exception { } } + /** + * Example command:
+ * opt/hadoop/bin/yarn resourcemanager -convert-fs-configuration
+ * -o /tmp/output
+ * -y /opt/hadoop/etc/hadoop/yarn-site.xml
+ * -f /opt/hadoop/etc/hadoop/fair-scheduler.xml
+ * -r /home/systest/sample-rules-config.properties
+ */ + @Test + @SuppressWarnings("checkstyle:javadocstyle") + public void testResourceManagerConvertFSConfigurationDefaults() + throws Exception { + setupFSConfigConversionFiles(); + + ArgumentCaptor conversionParams = + ArgumentCaptor.forClass(FSConfigToCSConfigConverterParams.class); + + final String mainSwitch = "-convert-fs-configuration"; + FSConfigToCSConfigConverter mockConverter = + mock(FSConfigToCSConfigConverter.class); + + ResourceManager.initFSArgumentHandler(mockConverter); + ResourceManager.main(new String[] {mainSwitch, "-o", OUTPUT_DIR, + "-y", YARN_SITE_XML, "-f", FS_ALLOC_FILE, "-r", + CONVERSION_RULES_FILE}); + + // validate params + verify(mockConverter).convert(conversionParams.capture()); + FSConfigToCSConfigConverterParams params = conversionParams.getValue(); + LOG.info("FS config converter parameters: " + params); + + assertThat(params.getYarnSiteXmlConfig()).isEqualTo(YARN_SITE_XML); + assertThat(params.getFairSchedulerXmlConfig()).isEqualTo(FS_ALLOC_FILE); + assertThat(params.getConversionRulesConfig()) + .isEqualTo(CONVERSION_RULES_FILE); + assertThat(params.isConsole()).isEqualTo(false); + } + + @Test + public void testResourceManagerConvertFSConfigurationWithConsoleParam() + throws Exception { + setupFSConfigConversionFiles(); + + ArgumentCaptor conversionParams = + ArgumentCaptor.forClass(FSConfigToCSConfigConverterParams.class); + + final String mainSwitch = "-convert-fs-configuration"; + FSConfigToCSConfigConverter mockConverter = + mock(FSConfigToCSConfigConverter.class); + + ResourceManager.initFSArgumentHandler(mockConverter); + ResourceManager.main(new String[] {mainSwitch, "-o", OUTPUT_DIR, + "-p", "-y", YARN_SITE_XML, "-f", FS_ALLOC_FILE, "-r", + CONVERSION_RULES_FILE}); + + // validate params + verify(mockConverter).convert(conversionParams.capture()); + FSConfigToCSConfigConverterParams params = conversionParams.getValue(); + LOG.info("FS config converter parameters: " + params); + + assertThat(params.getYarnSiteXmlConfig()).isEqualTo(YARN_SITE_XML); + assertThat(params.getFairSchedulerXmlConfig()).isEqualTo(FS_ALLOC_FILE); + assertThat(params.getConversionRulesConfig()) + .isEqualTo(CONVERSION_RULES_FILE); + assertThat(params.isConsole()).isEqualTo(true); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java new file mode 100644 index 0000000000..5fcf379e2b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintStream; +import java.io.PrintWriter; + +import static org.junit.Assert.assertTrue; + +/** + * Helper methods for FS->CS converter testing. + * + */ +public class FSConfigConverterTestCommons { + private final static String TEST_DIR = + new File(System.getProperty("test.build.data", "/tmp")).getAbsolutePath(); + public final static String FS_ALLOC_FILE = + new File(TEST_DIR, "test-fair-scheduler.xml").getAbsolutePath(); + public final static String YARN_SITE_XML = + new File(TEST_DIR, "test-yarn-site.xml").getAbsolutePath(); + public final static String CONVERSION_RULES_FILE = + new File(TEST_DIR, "test-conversion-rules.properties").getAbsolutePath(); + public final static String OUTPUT_DIR = + new File(TEST_DIR, "conversion-output").getAbsolutePath(); + + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); + private PrintStream originalOutStream; + private PrintStream originalErrStream; + + public FSConfigConverterTestCommons() { + saveOriginalStreams(); + replaceStreamsWithByteArrays(); + } + + public void setUp() throws IOException { + File d = new File(TEST_DIR, "conversion-output"); + if (d.exists()) { + FileUtils.deleteDirectory(d); + } + boolean success = d.mkdirs(); + assertTrue("Can't create directory: " + d.getAbsolutePath(), success); + } + + public void tearDown() { + deleteTestFiles(); + restoreStreams(); + } + + private void saveOriginalStreams() { + originalOutStream = System.out; + originalErrStream = System.err; + } + + private void replaceStreamsWithByteArrays() { + System.setOut(new PrintStream(outContent)); + System.setErr(new PrintStream(errContent)); + } + + private void restoreStreams() { + System.setOut(originalOutStream); + System.setErr(originalErrStream); + } + + ByteArrayOutputStream getErrContent() { + return errContent; + } + + private void deleteTestFiles() { + //Files may not be created so we are not strict here! + deleteFile(FS_ALLOC_FILE, false); + deleteFile(YARN_SITE_XML, false); + deleteFile(CONVERSION_RULES_FILE, false); + deleteFile(OUTPUT_DIR, false); + } + + private static void deleteFile(String f, boolean strict) { + boolean delete = new File(f).delete(); + if (strict && !delete) { + throw new RuntimeException("Can't delete test file: " + f); + } + } + + public static void setupFSConfigConversionFiles() throws IOException { + configureFairSchedulerXml(); + configureYarnSiteXmlWithFsAllocFileDefined(); + configureDummyConversionRulesFile(); + } + + @SuppressWarnings("checkstyle:linelength") + public static void configureFairSchedulerXml() throws IOException { + PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE)); + out.println(""); + out.println(""); + out.println("-1.0"); + out.println("fair"); + addQueue(out, ""); + out.println(""); + out.close(); + } + + @SuppressWarnings("checkstyle:linelength") + private static void addQueue(PrintWriter out, String additionalConfig) { + out.println(""); + out.println(" fair"); + out.println(" 1.0"); + out.println(" 100"); + out.println(" 120"); + out.println(" .5"); + + if (StringUtils.isNotEmpty(additionalConfig)) { + out.println(additionalConfig); + } + out.println(""); + } + + public static void configureEmptyFairSchedulerXml() throws IOException { + PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE)); + out.println(""); + out.println(""); + out.close(); + } + + public static void configureYarnSiteXmlWithFsAllocFileDefined() + throws IOException { + PrintWriter out = new PrintWriter(new FileWriter(YARN_SITE_XML)); + out.println(""); + out.println(""); + out.println("" + FairSchedulerConfiguration.ALLOCATION_FILE + + ""); + out.println("" + FS_ALLOC_FILE + ""); + out.println(""); + out.close(); + } + + public static void configureEmptyYarnSiteXml() throws IOException { + PrintWriter out = new PrintWriter(new FileWriter(YARN_SITE_XML)); + out.println(""); + out.println(""); + out.close(); + } + + public static void configureDummyConversionRulesFile() throws IOException { + PrintWriter out = new PrintWriter(new FileWriter(CONVERSION_RULES_FILE)); + out.println("dummy_key=dummy_value"); + out.close(); + } + + public static void configureInvalidConversionRulesFile() throws IOException { + PrintWriter out = new PrintWriter(new FileWriter(CONVERSION_RULES_FILE)); + out.println("bla"); + out.close(); + } + + public static void configureEmptyConversionRulesFile() throws IOException { + PrintWriter out = new PrintWriter(new FileWriter(CONVERSION_RULES_FILE)); + out.close(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java new file mode 100644 index 0000000000..70224fcc8a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java @@ -0,0 +1,379 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; + +import com.google.common.collect.Lists; +import org.apache.commons.cli.MissingOptionException; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Unit tests for FSConfigToCSConfigArgumentHandler. + * + */ +@RunWith(MockitoJUnitRunner.class) +public class TestFSConfigToCSConfigArgumentHandler { + private static final Logger LOG = + LoggerFactory.getLogger(TestFSConfigToCSConfigArgumentHandler.class); + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Mock + private FSConfigToCSConfigConverter mockConverter; + private FSConfigConverterTestCommons fsTestCommons; + + @Before + public void setUp() throws IOException { + fsTestCommons = new FSConfigConverterTestCommons(); + fsTestCommons.setUp(); + } + + @After + public void tearDown() { + fsTestCommons.tearDown(); + } + + private void setupFSConfigConversionFiles(boolean defineAllocationFile) + throws IOException { + FSConfigConverterTestCommons.configureFairSchedulerXml(); + + if (defineAllocationFile) { + FSConfigConverterTestCommons.configureYarnSiteXmlWithFsAllocFileDefined(); + } else { + FSConfigConverterTestCommons.configureEmptyYarnSiteXml(); + } + FSConfigConverterTestCommons.configureDummyConversionRulesFile(); + } + + + private FSConfigToCSConfigArgumentHandler createArgumentHandler() { + return new FSConfigToCSConfigArgumentHandler(mockConverter); + } + + private static String[] getDefaultArgumentsAsArray() { + List args = getDefaultArguments(); + return args.toArray(new String[0]); + } + + private static List getDefaultArguments() { + return Lists.newArrayList("-y", FSConfigConverterTestCommons.YARN_SITE_XML, + "-o", FSConfigConverterTestCommons.OUTPUT_DIR); + } + + private String[] getArgumentsAsArrayWithDefaults(String... args) { + List result = getDefaultArguments(); + result.addAll(Arrays.asList(args)); + return result.toArray(new String[0]); + } + + private String[] getArgumentsAsArray(String... args) { + List result = Lists.newArrayList(); + result.addAll(Arrays.asList(args)); + return result.toArray(new String[0]); + } + + @Test + public void testMissingYarnSiteXmlArgument() throws Exception { + setupFSConfigConversionFiles(true); + + FSConfigToCSConfigArgumentHandler argumentHandler = + createArgumentHandler(); + + String[] args = new String[] {"-o", + FSConfigConverterTestCommons.OUTPUT_DIR}; + + expectedException.expect(MissingOptionException.class); + expectedException.expectMessage("Missing required option: y"); + + argumentHandler.parseAndConvert(args); + } + + + @Test + public void testMissingFairSchedulerXmlArgument() throws Exception { + setupFSConfigConversionFiles(true); + FSConfigToCSConfigArgumentHandler argumentHandler = + createArgumentHandler(); + + argumentHandler.parseAndConvert(getDefaultArgumentsAsArray()); + } + + @Test + public void testMissingOutputDirArgument() throws Exception { + setupFSConfigConversionFiles(true); + + FSConfigToCSConfigArgumentHandler argumentHandler = + createArgumentHandler(); + + String[] args = new String[] {"-y", + FSConfigConverterTestCommons.YARN_SITE_XML}; + + expectedException.expect(MissingOptionException.class); + expectedException.expectMessage("Missing required option: o"); + + argumentHandler.parseAndConvert(args); + } + + @Test + public void testMissingRulesConfiguration() throws Exception { + setupFSConfigConversionFiles(true); + FSConfigToCSConfigArgumentHandler argumentHandler = + createArgumentHandler(); + + argumentHandler.parseAndConvert(getDefaultArgumentsAsArray()); + } + + @Test + public void testInvalidRulesConfigFile() throws Exception { + FSConfigConverterTestCommons.configureYarnSiteXmlWithFsAllocFileDefined(); + FSConfigConverterTestCommons.configureFairSchedulerXml(); + FSConfigConverterTestCommons.configureInvalidConversionRulesFile(); + + FSConfigToCSConfigArgumentHandler argumentHandler = + createArgumentHandler(); + + String[] args = getArgumentsAsArrayWithDefaults(); + argumentHandler.parseAndConvert(args); + } + + @Test + public void testInvalidOutputDir() throws Exception { + FSConfigConverterTestCommons.configureYarnSiteXmlWithFsAllocFileDefined(); + FSConfigConverterTestCommons.configureFairSchedulerXml(); + FSConfigConverterTestCommons.configureDummyConversionRulesFile(); + + FSConfigToCSConfigArgumentHandler argumentHandler = + createArgumentHandler(); + + String[] args = getArgumentsAsArray("-y", + FSConfigConverterTestCommons.YARN_SITE_XML, "-o", + FSConfigConverterTestCommons.YARN_SITE_XML); + + argumentHandler.parseAndConvert(args); + System.out.println(fsTestCommons.getErrContent()); + assertTrue("Error content missing", fsTestCommons.getErrContent() + .toString() + .contains("Cannot start FS config conversion due to the following " + + "precondition error")); + } + + @Test + public void testFairSchedulerXmlIsNotDefinedIfItsDefinedInYarnSiteXml() + throws Exception { + setupFSConfigConversionFiles(true); + FSConfigToCSConfigArgumentHandler argumentHandler = + createArgumentHandler(); + argumentHandler.parseAndConvert(getDefaultArgumentsAsArray()); + } + + @Test + public void testEmptyYarnSiteXmlSpecified() throws Exception { + FSConfigConverterTestCommons.configureFairSchedulerXml(); + FSConfigConverterTestCommons.configureEmptyYarnSiteXml(); + FSConfigConverterTestCommons.configureDummyConversionRulesFile(); + + FSConfigToCSConfigArgumentHandler argumentHandler = + createArgumentHandler(); + + String[] args = getArgumentsAsArrayWithDefaults("-f", + FSConfigConverterTestCommons.FS_ALLOC_FILE); + argumentHandler.parseAndConvert(args); + } + + @Test + public void testEmptyFairSchedulerXmlSpecified() throws Exception { + FSConfigConverterTestCommons.configureEmptyFairSchedulerXml(); + FSConfigConverterTestCommons.configureEmptyYarnSiteXml(); + FSConfigConverterTestCommons.configureDummyConversionRulesFile(); + + FSConfigToCSConfigArgumentHandler argumentHandler = + createArgumentHandler(); + + String[] args = getArgumentsAsArrayWithDefaults("-f", + FSConfigConverterTestCommons.FS_ALLOC_FILE); + argumentHandler.parseAndConvert(args); + } + + @Test + public void testEmptyRulesConfigurationSpecified() throws Exception { + FSConfigConverterTestCommons.configureEmptyFairSchedulerXml(); + FSConfigConverterTestCommons.configureEmptyYarnSiteXml(); + FSConfigConverterTestCommons.configureEmptyConversionRulesFile(); + + FSConfigToCSConfigArgumentHandler argumentHandler = + createArgumentHandler(); + + String[] args = getArgumentsAsArrayWithDefaults("-f", + FSConfigConverterTestCommons.FS_ALLOC_FILE, + "-r", FSConfigConverterTestCommons.CONVERSION_RULES_FILE); + argumentHandler.parseAndConvert(args); + } + + @Test + public void testConvertFSConfigurationDefaults() throws Exception { + setupFSConfigConversionFiles(true); + + ArgumentCaptor conversionParams = + ArgumentCaptor.forClass(FSConfigToCSConfigConverterParams.class); + + FSConfigToCSConfigArgumentHandler argumentHandler = + new FSConfigToCSConfigArgumentHandler(mockConverter); + + String[] args = getArgumentsAsArrayWithDefaults("-f", + FSConfigConverterTestCommons.FS_ALLOC_FILE, + "-r", FSConfigConverterTestCommons.CONVERSION_RULES_FILE); + argumentHandler.parseAndConvert(args); + + // validate params + Mockito.verify(mockConverter).convert(conversionParams.capture()); + FSConfigToCSConfigConverterParams params = conversionParams.getValue(); + LOG.info("FS config converter parameters: " + params); + + assertEquals("Yarn site config", + FSConfigConverterTestCommons.YARN_SITE_XML, + params.getYarnSiteXmlConfig()); + assertEquals("FS xml", FSConfigConverterTestCommons.FS_ALLOC_FILE, + params.getFairSchedulerXmlConfig()); + assertEquals("Conversion rules config", + FSConfigConverterTestCommons.CONVERSION_RULES_FILE, + params.getConversionRulesConfig()); + assertFalse("Console mode", params.isConsole()); + } + + @Test + public void testConvertFSConfigurationWithConsoleParam() + throws Exception { + setupFSConfigConversionFiles(true); + + ArgumentCaptor conversionParams = + ArgumentCaptor.forClass(FSConfigToCSConfigConverterParams.class); + + FSConfigToCSConfigArgumentHandler argumentHandler = + new FSConfigToCSConfigArgumentHandler(mockConverter); + + String[] args = getArgumentsAsArrayWithDefaults("-f", + FSConfigConverterTestCommons.FS_ALLOC_FILE, + "-r", FSConfigConverterTestCommons.CONVERSION_RULES_FILE, "-p"); + argumentHandler.parseAndConvert(args); + + // validate params + Mockito.verify(mockConverter).convert(conversionParams.capture()); + FSConfigToCSConfigConverterParams params = conversionParams.getValue(); + LOG.info("FS config converter parameters: " + params); + + assertEquals("Yarn site config", + FSConfigConverterTestCommons.YARN_SITE_XML, + params.getYarnSiteXmlConfig()); + assertEquals("FS xml", FSConfigConverterTestCommons.FS_ALLOC_FILE, + params.getFairSchedulerXmlConfig()); + assertEquals("Conversion rules config", + FSConfigConverterTestCommons.CONVERSION_RULES_FILE, + params.getConversionRulesConfig()); + assertTrue("Console mode", params.isConsole()); + } + + @Test + public void testConvertFSConfigurationClusterResource() + throws Exception { + setupFSConfigConversionFiles(true); + + ArgumentCaptor conversionParams = + ArgumentCaptor.forClass(FSConfigToCSConfigConverterParams.class); + + FSConfigToCSConfigArgumentHandler argumentHandler = + new FSConfigToCSConfigArgumentHandler(mockConverter); + + String[] args = getArgumentsAsArrayWithDefaults("-f", + FSConfigConverterTestCommons.FS_ALLOC_FILE, + "-r", FSConfigConverterTestCommons.CONVERSION_RULES_FILE, + "-p", "-c", "vcores=20, memory-mb=240"); + argumentHandler.parseAndConvert(args); + + // validate params + Mockito.verify(mockConverter).convert(conversionParams.capture()); + FSConfigToCSConfigConverterParams params = conversionParams.getValue(); + LOG.info("FS config converter parameters: " + params); + + assertEquals("Yarn site config", + FSConfigConverterTestCommons.YARN_SITE_XML, + params.getYarnSiteXmlConfig()); + assertEquals("FS xml", + FSConfigConverterTestCommons.FS_ALLOC_FILE, + params.getFairSchedulerXmlConfig()); + assertEquals("Conversion rules config", + FSConfigConverterTestCommons.CONVERSION_RULES_FILE, + params.getConversionRulesConfig()); + assertEquals("Cluster resource", "vcores=20, memory-mb=240", + params.getClusterResource()); + assertTrue("Console mode", params.isConsole()); + } + + @Test + public void testConvertFSConfigurationErrorHandling() throws Exception { + setupFSConfigConversionFiles(true); + + String[] args = getArgumentsAsArrayWithDefaults("-f", + FSConfigConverterTestCommons.FS_ALLOC_FILE, + "-r", FSConfigConverterTestCommons.CONVERSION_RULES_FILE, "-p"); + FSConfigToCSConfigArgumentHandler argumentHandler = + createArgumentHandler(); + + Mockito.doThrow(UnsupportedPropertyException.class) + .when(mockConverter) + .convert(ArgumentMatchers.any(FSConfigToCSConfigConverterParams.class)); + argumentHandler.parseAndConvert(args); + assertTrue("Error content missing", fsTestCommons.getErrContent() + .toString().contains("Unsupported property/setting encountered")); + } + + @Test + public void testConvertFSConfigurationErrorHandling2() throws Exception { + setupFSConfigConversionFiles(true); + + String[] args = getArgumentsAsArrayWithDefaults("-f", + FSConfigConverterTestCommons.FS_ALLOC_FILE, + "-r", FSConfigConverterTestCommons.CONVERSION_RULES_FILE, "-p"); + FSConfigToCSConfigArgumentHandler argumentHandler = + createArgumentHandler(); + + Mockito.doThrow(ConversionException.class).when(mockConverter) + .convert(ArgumentMatchers.any(FSConfigToCSConfigConverterParams.class)); + argumentHandler.parseAndConvert(args); + assertTrue("Error content missing", fsTestCommons.getErrContent() + .toString().contains("Fatal error during FS config conversion")); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java new file mode 100644 index 0000000000..bdcfca73ca --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java @@ -0,0 +1,460 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; + +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.DYNAMIC_MAX_ASSIGN; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.MAX_CAPACITY_PERCENTAGE; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.MAX_CHILD_CAPACITY; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.QUEUE_AUTO_CREATE; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.RESERVATION_SYSTEM; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.SPECIFIED_NOT_FIRST; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.USER_MAX_APPS_DEFAULT; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.USER_MAX_RUNNING_APPS; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.RuleAction.ABORT; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.util.Map; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + + +/** + * Unit tests for FSConfigToCSConfigConverter. + * + */ +@RunWith(MockitoJUnitRunner.class) +public class TestFSConfigToCSConfigConverter { + private static final Resource CLUSTER_RESOURCE = + Resource.newInstance(16384, 16); + private static final String FILE_PREFIX = "file:"; + private static final String FAIR_SCHEDULER_XML = + prepareFileName("fair-scheduler-conversion.xml"); + + @Mock + private FSConfigToCSConfigRuleHandler ruleHandler; + + private FSConfigToCSConfigConverter converter; + private Configuration config; + + private ByteArrayOutputStream csConfigOut; + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + private FSConfigConverterTestCommons converterTestCommons; + + private static String prepareFileName(String f) { + return FILE_PREFIX + new File("src/test/resources/" + f).getAbsolutePath(); + } + + private static final String FAIR_SCHEDULER_XML_INVALID = + prepareFileName("fair-scheduler-invalid.xml"); + private static final String YARN_SITE_XML = + prepareFileName("yarn-site-with-allocation-file-ref.xml"); + private static final String YARN_SITE_XML_NO_REF_TO_FS_XML = + prepareFileName("yarn-site.xml"); + private static final String YARN_SITE_XML_INVALID = + prepareFileName("yarn-site-with-invalid-allocation-file-ref.xml"); + private static final String CONVERSION_RULES_FILE = + new File("src/test/resources/conversion-rules.properties") + .getAbsolutePath(); + + @Before + public void setup() throws IOException { + config = new Configuration(false); + config.set(FairSchedulerConfiguration.ALLOCATION_FILE, FAIR_SCHEDULER_XML); + config.setBoolean(FairSchedulerConfiguration.MIGRATION_MODE, true); + createConverter(); + converterTestCommons = new FSConfigConverterTestCommons(); + converterTestCommons.setUp(); + } + + @After + public void tearDown() { + converterTestCommons.tearDown(); + } + + private void createConverter() { + converter = new FSConfigToCSConfigConverter(ruleHandler); + converter.setClusterResource(CLUSTER_RESOURCE); + ByteArrayOutputStream yarnSiteOut = new ByteArrayOutputStream(); + csConfigOut = new ByteArrayOutputStream(); + + converter.setCapacitySchedulerConfigOutputStream(csConfigOut); + converter.setYarnSiteOutputStream(yarnSiteOut); + } + + private FSConfigToCSConfigConverterParams.Builder + createDefaultParamsBuilder() { + return FSConfigToCSConfigConverterParams.Builder.create() + .withYarnSiteXmlConfig(YARN_SITE_XML) + .withOutputDirectory(FSConfigConverterTestCommons.OUTPUT_DIR); + } + + private FSConfigToCSConfigConverterParams.Builder + createParamsBuilder(String yarnSiteConfig) { + return FSConfigToCSConfigConverterParams.Builder.create() + .withYarnSiteXmlConfig(yarnSiteConfig) + .withOutputDirectory(FSConfigConverterTestCommons.OUTPUT_DIR); + } + + @Test + public void testDefaultMaxApplications() throws Exception { + converter.convert(config); + + Configuration conf = getConvertedCSConfig(); + int maxApps = + conf.getInt( + CapacitySchedulerConfiguration.MAXIMUM_SYSTEM_APPLICATIONS, -1); + + assertEquals("Default max apps", 15, maxApps); + } + + @Test + public void testDefaultMaxAMShare() throws Exception { + converter.convert(config); + + Configuration conf = getConvertedCSConfig(); + String maxAmShare = + conf.get(CapacitySchedulerConfiguration. + MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT); + + assertEquals("Default max AM share", "0.16", maxAmShare); + } + + @Test + public void testConvertACLs() throws Exception { + converter.convert(config); + + Configuration conf = getConvertedCSConfig(); + + // root + assertEquals("root submit ACL", "alice,bob,joe,john hadoop_users", + conf.get(PREFIX + "root.acl_submit_applications")); + assertEquals("root admin ACL", "alice,bob,joe,john hadoop_users", + conf.get(PREFIX + "root.acl_administer_queue")); + + // root.admins.bob + assertEquals("root.admins.bob submit ACL", "bob ", + conf.get(PREFIX + "root.admins.bob.acl_submit_applications")); + assertEquals("root.admins.bob admin ACL", "bob ", + conf.get(PREFIX + "root.admins.bob.acl_administer_queue")); + + // root.admins.alice + assertEquals("root.admins.alice submit ACL", "alice ", + conf.get(PREFIX + "root.admins.alice.acl_submit_applications")); + assertEquals("root.admins.alice admin ACL", "alice ", + conf.get(PREFIX + "root.admins.alice.acl_administer_queue")); + + // root.users.john + assertEquals("root.users.john submit ACL", "john ", + conf.get(PREFIX + "root.users.john.acl_submit_applications")); + assertEquals("root.users.john admin ACL", "john ", + conf.get(PREFIX + "root.users.john.acl_administer_queue")); + + // root.users.joe + assertEquals("root.users.joe submit ACL", "joe ", + conf.get(PREFIX + "root.users.joe.acl_submit_applications")); + assertEquals("root.users.joe admin ACL", "joe ", + conf.get(PREFIX + "root.users.joe.acl_administer_queue")); + } + + @Test + public void testDefaultMaxRunningApps() throws Exception { + converter.convert(config); + + Configuration conf = getConvertedCSConfig(); + + // default setting + assertEquals("Default max apps", 15, + conf.getInt(PREFIX + "maximum-applications", -1)); + } + + @Test + public void testMixedQueueOrderingPolicy() throws Exception { + expectedException.expect(ConversionException.class); + expectedException.expectMessage( + "DRF ordering policy cannot be used together with fifo/fair"); + String absolutePath = + new File("src/test/resources/fair-scheduler-orderingpolicy-mixed.xml") + .getAbsolutePath(); + config.set(FairSchedulerConfiguration.ALLOCATION_FILE, + FILE_PREFIX + absolutePath); + + converter.convert(config); + } + + @Test + public void testQueueMaxChildCapacityNotSupported() throws Exception { + expectedException.expect(UnsupportedPropertyException.class); + expectedException.expectMessage("test"); + + Mockito.doThrow(new UnsupportedPropertyException("test")) + .when(ruleHandler).handleMaxChildCapacity(); + + converter.convert(config); + } + + @Test + public void testReservationSystemNotSupported() throws Exception { + expectedException.expect(UnsupportedPropertyException.class); + expectedException.expectMessage("maxCapacity"); + + Mockito.doThrow(new UnsupportedPropertyException("maxCapacity")) + .when(ruleHandler).handleMaxChildCapacity(); + config.setBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE, true); + + converter.convert(config); + } + + @Test + public void testUserMaxAppsNotSupported() throws Exception { + expectedException.expect(UnsupportedPropertyException.class); + expectedException.expectMessage("userMaxApps"); + + Mockito.doThrow(new UnsupportedPropertyException("userMaxApps")) + .when(ruleHandler).handleUserMaxApps(); + + converter.convert(config); + } + + @Test + public void testUserMaxAppsDefaultNotSupported() throws Exception { + expectedException.expect(UnsupportedPropertyException.class); + expectedException.expectMessage("userMaxAppsDefault"); + + Mockito.doThrow(new UnsupportedPropertyException("userMaxAppsDefault")) + .when(ruleHandler).handleUserMaxAppsDefault(); + + converter.convert(config); + } + + @Test + public void testConvertFSConfigurationClusterResource() throws Exception { + FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder() + .withClusterResource("vcores=20, memory-mb=240") + .build(); + converter.convert(params); + assertEquals("Resource", Resource.newInstance(240, 20), + converter.getClusterResource()); + } + + @Test + public void testConvertFSConfigPctModeUsedAndClusterResourceDefined() + throws Exception { + FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder() + .withClusterResource("vcores=20, memory-mb=240") + .build(); + converter.convert(params); + assertEquals("Resource", Resource.newInstance(240, 20), + converter.getClusterResource()); + } + + @Test + public void testConvertFSConfigPctModeUsedAndClusterResourceNotDefined() + throws Exception { + FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder() + .build(); + + expectedException.expect(ConversionException.class); + expectedException.expectMessage("cluster resource parameter" + + " is not defined via CLI"); + + converter.convert(params); + } + + @Test + public void testConvertFSConfigurationClusterResourceInvalid() + throws Exception { + FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder() + .withClusterResource("vcores=20, memory-mb=240G") + .build(); + + expectedException.expect(ConversionException.class); + expectedException.expectMessage("Error while parsing resource"); + + converter.convert(params); + } + + @Test + public void testConvertFSConfigurationClusterResourceInvalid2() + throws Exception { + FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder() + .withClusterResource("vcores=20, memmmm=240") + .build(); + + expectedException.expect(ConversionException.class); + expectedException.expectMessage("Error while parsing resource"); + + converter.convert(params); + } + + @Test + public void testConvertFSConfigurationRulesFile() throws Exception { + ruleHandler = new FSConfigToCSConfigRuleHandler(); + createConverter(); + + FSConfigToCSConfigConverterParams params = + createDefaultParamsBuilder() + .withConversionRulesConfig(CONVERSION_RULES_FILE) + .withClusterResource("vcores=20, memory-mb=2400") + .build(); + + try { + converter.convert(params); + fail("Should have thrown UnsupportedPropertyException!"); + } catch (UnsupportedPropertyException e) { + //need to catch exception so we can check the rules + } + + ruleHandler = converter.getRuleHandler(); + Map actions = + ruleHandler.getActions(); + + assertEquals("maxCapacityPercentage", + ABORT, actions.get(MAX_CAPACITY_PERCENTAGE)); + assertEquals("maxChildCapacity", + ABORT, actions.get(MAX_CHILD_CAPACITY)); + assertEquals("userMaxRunningApps", + ABORT, actions.get(USER_MAX_RUNNING_APPS)); + assertEquals("userMaxAppsDefault", + ABORT, actions.get(USER_MAX_APPS_DEFAULT)); + assertEquals("dynamicMaxAssign", + ABORT, actions.get(DYNAMIC_MAX_ASSIGN)); + assertEquals("specifiedNotFirstRule", + ABORT, actions.get(SPECIFIED_NOT_FIRST)); + assertEquals("reservationSystem", + ABORT, actions.get(RESERVATION_SYSTEM)); + assertEquals("queueAutoCreate", + ABORT, actions.get(QUEUE_AUTO_CREATE)); + } + + @Test + public void testConvertFSConfigurationUndefinedYarnSiteConfig() + throws Exception { + FSConfigToCSConfigConverterParams params = + FSConfigToCSConfigConverterParams.Builder.create() + .withYarnSiteXmlConfig(null) + .withOutputDirectory(FSConfigConverterTestCommons.OUTPUT_DIR) + .build(); + + expectedException.expect(PreconditionException.class); + expectedException.expectMessage( + "yarn-site.xml configuration is not defined"); + + converter.convert(params); + } + + @Test + public void testConvertCheckOutputDir() throws Exception { + FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder() + .withClusterResource("vcores=20, memory-mb=240") + .build(); + + converter.convert(params); + + Configuration conf = + getConvertedCSConfig(FSConfigConverterTestCommons.OUTPUT_DIR); + + File capacityFile = new File(FSConfigConverterTestCommons.OUTPUT_DIR, + "capacity-scheduler.xml"); + assertTrue("Capacity file exists", capacityFile.exists()); + assertTrue("Capacity file length > 0", capacityFile.length() > 0); + assertTrue("No. of configuration elements > 0", conf.size() > 0); + + File yarnSiteFile = new File(FSConfigConverterTestCommons.OUTPUT_DIR, + "yarn-site.xml"); + assertTrue("Yarn site exists", yarnSiteFile.exists()); + assertTrue("Yarn site length > 0", yarnSiteFile.length() > 0); + } + + @Test + public void testFairSchedulerXmlIsNotDefinedNeitherDirectlyNorInYarnSiteXml() + throws Exception { + FSConfigToCSConfigConverterParams params = + createParamsBuilder(YARN_SITE_XML_NO_REF_TO_FS_XML) + .withClusterResource("vcores=20, memory-mb=240") + .build(); + + expectedException.expect(PreconditionException.class); + expectedException.expectMessage("fair-scheduler.xml is not defined"); + converter.convert(params); + } + + @Test + public void testInvalidFairSchedulerXml() throws Exception { + FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder() + .withClusterResource("vcores=20, memory-mb=240") + .withFairSchedulerXmlConfig(FAIR_SCHEDULER_XML_INVALID) + .build(); + + expectedException.expect(RuntimeException.class); + converter.convert(params); + } + + @Test + public void testInvalidYarnSiteXml() throws Exception { + FSConfigToCSConfigConverterParams params = + createParamsBuilder(YARN_SITE_XML_INVALID) + .withClusterResource("vcores=20, memory-mb=240") + .build(); + + expectedException.expect(RuntimeException.class); + converter.convert(params); + } + + private Configuration getConvertedCSConfig() { + ByteArrayInputStream input = + new ByteArrayInputStream(csConfigOut.toByteArray()); + assertTrue("CS config output has length of 0!", + csConfigOut.toByteArray().length > 0); + Configuration conf = new Configuration(false); + conf.addResource(input); + + return conf; + } + + private Configuration getConvertedCSConfig(String dir) throws IOException { + File capacityFile = new File(dir, "capacity-scheduler.xml"); + ByteArrayInputStream input = + new ByteArrayInputStream(FileUtils.readFileToByteArray(capacityFile)); + Configuration conf = new Configuration(false); + conf.addResource(input); + + return conf; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigRuleHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigRuleHandler.java new file mode 100644 index 0000000000..72f53afa08 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigRuleHandler.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; + +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.DYNAMIC_MAX_ASSIGN; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.MAX_CAPACITY_PERCENTAGE; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.MAX_CHILD_CAPACITY; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.MAX_CHILD_QUEUE_LIMIT; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.QUEUE_AUTO_CREATE; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.RESERVATION_SYSTEM; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.SPECIFIED_NOT_FIRST; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.USER_MAX_APPS_DEFAULT; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigRuleHandler.USER_MAX_RUNNING_APPS; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.Properties; +import org.junit.Test; + +/** + * Unit tests for FSConfigToCSConfigRuleHandler. + * + */ +public class TestFSConfigToCSConfigRuleHandler { + private static final String ABORT = "abort"; + private static final String WARNING = "warning"; + + private FSConfigToCSConfigRuleHandler ruleHandler; + + @Test + public void testInitPropertyActionsToWarning() throws IOException { + ruleHandler = new FSConfigToCSConfigRuleHandler(new Properties()); + + ruleHandler.handleChildQueueCount("test", 1); + ruleHandler.handleDynamicMaxAssign(); + ruleHandler.handleMaxCapacityPercentage("test"); + ruleHandler.handleMaxChildCapacity(); + ruleHandler.handleQueueAutoCreate("test"); + ruleHandler.handleReservationSystem(); + ruleHandler.handleSpecifiedNotFirstRule(); + ruleHandler.handleUserMaxApps(); + ruleHandler.handleUserMaxAppsDefault(); + } + + @Test + public void testAllRulesWarning() throws IOException { + Properties rules = new Properties(); + rules.put(DYNAMIC_MAX_ASSIGN, WARNING); + rules.put(MAX_CAPACITY_PERCENTAGE, WARNING); + rules.put(MAX_CHILD_CAPACITY, WARNING); + rules.put(QUEUE_AUTO_CREATE, WARNING); + rules.put(RESERVATION_SYSTEM, WARNING); + rules.put(SPECIFIED_NOT_FIRST, WARNING); + rules.put(USER_MAX_APPS_DEFAULT, WARNING); + rules.put(USER_MAX_RUNNING_APPS, WARNING); + + ruleHandler = new FSConfigToCSConfigRuleHandler(rules); + + ruleHandler.handleDynamicMaxAssign(); + ruleHandler.handleMaxCapacityPercentage("test"); + ruleHandler.handleMaxChildCapacity(); + ruleHandler.handleQueueAutoCreate("test"); + ruleHandler.handleReservationSystem(); + ruleHandler.handleSpecifiedNotFirstRule(); + ruleHandler.handleUserMaxApps(); + ruleHandler.handleUserMaxAppsDefault(); + } + + @Test + public void testAllRulesAbort() throws IOException { + Properties rules = new Properties(); + rules.put(DYNAMIC_MAX_ASSIGN, ABORT); + rules.put(MAX_CAPACITY_PERCENTAGE, ABORT); + rules.put(MAX_CHILD_CAPACITY, ABORT); + rules.put(QUEUE_AUTO_CREATE, ABORT); + rules.put(RESERVATION_SYSTEM, ABORT); + rules.put(SPECIFIED_NOT_FIRST, ABORT); + rules.put(USER_MAX_APPS_DEFAULT, ABORT); + rules.put(USER_MAX_RUNNING_APPS, ABORT); + rules.put(MAX_CHILD_QUEUE_LIMIT, "1"); + + ruleHandler = new FSConfigToCSConfigRuleHandler(rules); + + expectAbort(() -> ruleHandler.handleChildQueueCount("test", 2), + ConversionException.class); + expectAbort(() -> ruleHandler.handleDynamicMaxAssign()); + expectAbort(() -> ruleHandler.handleMaxCapacityPercentage("test")); + expectAbort(() -> ruleHandler.handleMaxChildCapacity()); + expectAbort(() -> ruleHandler.handleQueueAutoCreate("test")); + expectAbort(() -> ruleHandler.handleReservationSystem()); + expectAbort(() -> ruleHandler.handleSpecifiedNotFirstRule()); + expectAbort(() -> ruleHandler.handleUserMaxApps()); + expectAbort(() -> ruleHandler.handleUserMaxAppsDefault()); + } + + @Test(expected = ConversionException.class) + public void testMaxChildQueueCountNotInteger() throws IOException { + Properties rules = new Properties(); + rules.put(MAX_CHILD_QUEUE_LIMIT, "abc"); + + ruleHandler = new FSConfigToCSConfigRuleHandler(rules); + + ruleHandler.handleChildQueueCount("test", 1); + } + + private void expectAbort(VoidCall call) { + expectAbort(call, UnsupportedPropertyException.class); + } + + private void expectAbort(VoidCall call, Class exceptionClass) { + boolean exceptionThrown = false; + Throwable thrown = null; + + try { + call.apply(); + } catch (Throwable t) { + thrown = t; + exceptionThrown = true; + } + + assertTrue("Exception was not thrown", exceptionThrown); + assertEquals("Unexpected exception", exceptionClass, thrown.getClass()); + } + + @FunctionalInterface + private interface VoidCall { + void apply(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java new file mode 100644 index 0000000000..4e8aad8b5d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java @@ -0,0 +1,427 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; + +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.util.Set; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; +import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.google.common.collect.Sets; + + +/** + * Unit tests for FSQueueConverter. + * + */ +@RunWith(MockitoJUnitRunner.class) +public class TestFSQueueConverter { + private static final Resource CLUSTER_RESOURCE = + Resource.newInstance(16384, 16); + private final static Set ALL_QUEUES = + Sets.newHashSet("root", + "root.default", + "root.admins", + "root.users", + "root.admins.alice", + "root.admins.bob", + "root.users.joe", + "root.users.john"); + + private static final String FILE_PREFIX = "file:"; + private static final String FAIR_SCHEDULER_XML = + prepareFileName("fair-scheduler-conversion.xml"); + + private static String prepareFileName(String f) { + return FILE_PREFIX + new File("src/test/resources/" + f).getAbsolutePath(); + } + + private FSQueueConverter converter; + private Configuration config; + private Configuration csConfig; + private FairScheduler fs; + private FSQueue rootQueue; + + @Mock + private FSConfigToCSConfigRuleHandler ruleHandler; + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Before + public void setup() { + config = new Configuration(false); + config.set(FairSchedulerConfiguration.ALLOCATION_FILE, FAIR_SCHEDULER_XML); + config.setBoolean(FairSchedulerConfiguration.MIGRATION_MODE, true); + csConfig = new Configuration(false); + + fs = createFairScheduler(); + + createConverter(); + rootQueue = fs.getQueueManager().getRootQueue(); + } + + @After + public void tearDown() throws IOException { + if (fs != null) { + fs.close(); + } + } + + private FairScheduler createFairScheduler() { + RMContext ctx = new RMContextImpl(); + PlacementManager placementManager = new PlacementManager(); + ctx.setQueuePlacementManager(placementManager); + + FairScheduler fairScheduler = new FairScheduler(); + fairScheduler.setRMContext(ctx); + fairScheduler.init(config); + + return fairScheduler; + } + + private void createConverter() { + converter = new FSQueueConverter(ruleHandler, + csConfig, + false, + false, + false, + CLUSTER_RESOURCE, + 0.16f, + 15); + } + + @Test + public void testConvertQueueHierarchy() { + converter.convertQueueHierarchy(rootQueue); + + // root children + assertEquals("root children", "default,admins,users", + csConfig.get(PREFIX + "root.queues")); + + // root.admins children + assertEquals("root.admins children", "bob,alice", + csConfig.get(PREFIX + "root.admins.queues")); + + // root.default children - none + assertNull("root.default children", csConfig.get(PREFIX + "root.default" + + ".queues")); + + // root.users children + assertEquals("root.users children", "john,joe", + csConfig.get(PREFIX + "root.users.queues")); + + Set leafs = Sets.difference(ALL_QUEUES, + Sets.newHashSet("root", + "root.default", + "root.admins", + "root.users")); + + assertNoValueForQueues(leafs, ".queues", csConfig); + } + + @Test + public void testConvertQueueHierarchyWithSameLeafQueues() throws Exception { + expectedException.expect(ConversionException.class); + expectedException.expectMessage("Leaf queues must be unique"); + + String absolutePath = + new File("src/test/resources/fair-scheduler-sameleafqueue.xml") + .getAbsolutePath(); + config.set(FairSchedulerConfiguration.ALLOCATION_FILE, + FILE_PREFIX + absolutePath); + fs.close(); + fs = createFairScheduler(); + rootQueue = fs.getQueueManager().getRootQueue(); + + converter.convertQueueHierarchy(rootQueue); + } + + @Test + public void testQueueMaxAMShare() { + converter.convertQueueHierarchy(rootQueue); + + // root.admins.bob + assertEquals("root.admins.bob AM share", "1.0", + csConfig.get(PREFIX + "root.admins.bob.maximum-am-resource-percent")); + + // root.admins.alice + assertEquals("root.admins.alice AM share", "0.15", + csConfig.get(PREFIX + + "root.admins.alice.maximum-am-resource-percent")); + + Set remaining = Sets.difference(ALL_QUEUES, + Sets.newHashSet("root.admins.bob", "root.admins.alice")); + assertNoValueForQueues(remaining, ".maximum-am-resource-percent", + csConfig); + } + + @Test + public void testQueueMaxRunningApps() { + converter.convertQueueHierarchy(rootQueue); + + assertEquals("root.admins.alice max apps", 2, + csConfig.getInt(PREFIX + "root.admins.alice.maximum-applications", + -1)); + + Set remaining = Sets.difference(ALL_QUEUES, + Sets.newHashSet("root.admins.alice")); + assertNoValueForQueues(remaining, ".maximum-applications", csConfig); + } + + @Test + public void testQueueMaxAllocations() { + converter.convertQueueHierarchy(rootQueue); + + // root.admins vcores + mb + assertEquals("root.admins max vcores", 3, + csConfig.getInt(PREFIX + "root.admins.maximum-allocation-vcores", -1)); + assertEquals("root.admins max memory", 4096, + csConfig.getInt(PREFIX + "root.admins.maximum-allocation-mb", -1)); + + // root.users.john max vcores + mb + assertEquals("root.users.john max vcores", 2, + csConfig.getInt(PREFIX + "root.users.john.maximum-allocation-vcores", + -1)); + assertEquals("root.users.john max memory", 8192, + csConfig.getInt(PREFIX + "root.users.john.maximum-allocation-mb", -1)); + + Set remaining = Sets.difference(ALL_QUEUES, + Sets.newHashSet("root.admins", "root.users.john")); + assertNoValueForQueues(remaining, ".maximum-allocation-vcores", csConfig); + assertNoValueForQueues(remaining, ".maximum-allocation-mb", csConfig); + } + + @Test + public void testQueuePreemptionDisabled() { + converter = new FSQueueConverter(ruleHandler, + csConfig, + true, + false, + false, + CLUSTER_RESOURCE, + 0.16f, + 15); + + converter.convertQueueHierarchy(rootQueue); + + assertTrue("root.admins.alice preemption setting", + csConfig.getBoolean(PREFIX + "root.admins.alice.disable_preemption", + false)); + assertTrue("root.users.joe preemption setting", + csConfig.getBoolean(PREFIX + "root.users.joe.disable_preemption", + false)); + + Set remaining = Sets.difference(ALL_QUEUES, + Sets.newHashSet("root.admins.alice", "root.users.joe")); + assertNoValueForQueues(remaining, ".disable_preemption", csConfig); + } + + @Test + public void testQueuePreemptionDisabledWhenGlobalPreemptionDisabled() { + converter.convertQueueHierarchy(rootQueue); + + assertNoValueForQueues(ALL_QUEUES, ".disable_preemption", csConfig); + } + + @Test + public void testChildCapacity() { + converter.convertQueueHierarchy(rootQueue); + + // root + assertEquals("root.default capacity", "33.333", + csConfig.get(PREFIX + "root.default.capacity")); + assertEquals("root.admins capacity", "33.333", + csConfig.get(PREFIX + "root.admins.capacity")); + assertEquals("root.users capacity", "66.667", + csConfig.get(PREFIX + "root.users.capacity")); + + // root.users + assertEquals("root.users.john capacity", "25.000", + csConfig.get(PREFIX + "root.users.john.capacity")); + assertEquals("root.users.joe capacity", "75.000", + csConfig.get(PREFIX + "root.users.joe.capacity")); + + // root.admins + assertEquals("root.admins.alice capacity", "75.000", + csConfig.get(PREFIX + "root.admins.alice.capacity")); + assertEquals("root.admins.bob capacity", "25.000", + csConfig.get(PREFIX + "root.admins.bob.capacity")); + } + + @Test + public void testQueueMaximumCapacity() { + converter.convertQueueHierarchy(rootQueue); + + assertEquals("root.users.joe maximum capacity", "[memory=8192, vcores=8]", + csConfig.get(PREFIX + "root.users.joe.maximum-capacity")); + assertEquals("root.admins.bob maximum capacity", "[memory=8192, vcores=2]", + csConfig.get(PREFIX + "root.admins.bob.maximum-capacity")); + assertEquals("root.admins.alice maximum capacity", + "[memory=16384, vcores=4]", + csConfig.get(PREFIX + "root.admins.alice.maximum-capacity")); + + Set remaining = Sets.difference(ALL_QUEUES, + Sets.newHashSet("root.users.joe", + "root.admins.bob", + "root.admins.alice")); + assertNoValueForQueues(remaining, ".maximum-capacity", csConfig); + } + + @Test + public void testQueueAutoCreateChildQueue() { + config.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, true); + converter = new FSQueueConverter(ruleHandler, + csConfig, + false, + false, + true, + CLUSTER_RESOURCE, + 0.16f, + 15); + + converter.convertQueueHierarchy(rootQueue); + + assertTrueForQueues(ALL_QUEUES, ".auto-create-child-queue.enabled", + csConfig); + } + + @Test + public void testQueueSizeBasedWeightEnabled() { + converter = new FSQueueConverter(ruleHandler, + csConfig, + false, + true, + false, + CLUSTER_RESOURCE, + 0.16f, + 15); + + converter.convertQueueHierarchy(rootQueue); + + assertTrueForQueues(ALL_QUEUES, + ".ordering-policy.fair.enable-size-based-weight", csConfig); + } + + @Test + public void testQueueSizeBasedWeightDisabled() { + converter.convertQueueHierarchy(rootQueue); + + assertNoValueForQueues(ALL_QUEUES, + ".ordering-policy.fair.enable-size-based-weight", csConfig); + } + + @Test + public void testQueueOrderingPolicy() throws Exception { + String absolutePath = + new File("src/test/resources/fair-scheduler-orderingpolicy.xml") + .getAbsolutePath(); + config.set(FairSchedulerConfiguration.ALLOCATION_FILE, + FILE_PREFIX + absolutePath); + fs.close(); + fs = createFairScheduler(); + rootQueue = fs.getQueueManager().getRootQueue(); + + converter.convertQueueHierarchy(rootQueue); + + // root + assertEquals("root ordering policy", "fair", + csConfig.get(PREFIX + "root.ordering-policy")); + assertEquals("root.default ordering policy", "fair", + csConfig.get(PREFIX + "root.default.ordering-policy")); + assertEquals("root.admins ordering policy", "fair", + csConfig.get(PREFIX + "root.admins.ordering-policy")); + assertEquals("root.users ordering policy", "fair", + csConfig.get(PREFIX + "root.users.ordering-policy")); + + // root.users + assertEquals("root.users.joe ordering policy", "fair", + csConfig.get(PREFIX + "root.users.joe.ordering-policy")); + assertEquals("root.users.john ordering policy", "FIFO", + csConfig.get(PREFIX + "root.users.john.ordering-policy")); + + // root.admins + assertEquals("root.admins.alice ordering policy", "FIFO", + csConfig.get(PREFIX + "root.admins.alice.ordering-policy")); + assertEquals("root.admins.bob ordering policy", "fair", + csConfig.get(PREFIX + "root.admins.bob.ordering-policy")); + } + + @Test + public void testQueueMaxChildCapacityNotSupported() { + expectedException.expect(UnsupportedPropertyException.class); + expectedException.expectMessage("test"); + + Mockito.doThrow(new UnsupportedPropertyException("test")) + .when(ruleHandler).handleMaxChildCapacity(); + + converter.convertQueueHierarchy(rootQueue); + } + + @Test + public void testReservationSystemNotSupported() { + expectedException.expect(UnsupportedPropertyException.class); + expectedException.expectMessage("maxCapacity"); + + Mockito.doThrow(new UnsupportedPropertyException("maxCapacity")) + .when(ruleHandler).handleMaxChildCapacity(); + config.setBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE, true); + + converter.convertQueueHierarchy(rootQueue); + } + + private void assertNoValueForQueues(Set queues, String postfix, + Configuration config) { + for (String queue : queues) { + String key = PREFIX + queue + postfix; + assertNull("Key " + key + " has value, but it should be null", + config.get(key)); + } + } + + private void assertTrueForQueues(Set queues, String postfix, + Configuration config) { + for (String queue : queues) { + String key = PREFIX + queue + postfix; + assertTrue("Key " + key + " is false, should be true", + config.getBoolean(key, false)); + } + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java new file mode 100644 index 0000000000..edb9a4e9ca --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSYarnSiteConverter.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Unit tests for FSYarnSiteConverter. + * + */ +public class TestFSYarnSiteConverter { + private Configuration yarnConfig; + private FSYarnSiteConverter converter; + private Configuration yarnConvertedConfig; + + @Before + public void setup() { + yarnConfig = new Configuration(false); + yarnConvertedConfig = new Configuration(false); + converter = new FSYarnSiteConverter(); + } + + @SuppressWarnings("deprecation") + @Test + public void testSiteContinuousSchedulingConversion() { + yarnConfig.setBoolean( + FairSchedulerConfiguration.CONTINUOUS_SCHEDULING_ENABLED, true); + yarnConfig.setInt( + FairSchedulerConfiguration.CONTINUOUS_SCHEDULING_SLEEP_MS, 666); + + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig); + + assertTrue("Cont. scheduling", yarnConvertedConfig.getBoolean( + CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE, false)); + assertEquals("Scheduling interval", 666, + yarnConvertedConfig.getInt( + "yarn.scheduler.capacity.schedule-asynchronously" + + ".scheduling-interval-ms", -1)); + } + + @Test + public void testSiteMinimumAllocationIncrementConversion() { + yarnConfig.setInt("yarn.resource-types.memory-mb.increment-allocation", 11); + yarnConfig.setInt("yarn.resource-types.vcores.increment-allocation", 5); + + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig); + + assertEquals("Memory alloc increment", 11, + yarnConvertedConfig.getInt("yarn.scheduler.minimum-allocation-mb", + -1)); + assertEquals("Vcore increment", 5, + yarnConvertedConfig.getInt("yarn.scheduler.minimum-allocation-vcores", + -1)); + } + + @Test + public void testSitePreemptionConversion() { + yarnConfig.setBoolean(FairSchedulerConfiguration.PREEMPTION, true); + yarnConfig.setInt(FairSchedulerConfiguration.WAIT_TIME_BEFORE_KILL, 123); + yarnConfig.setInt( + FairSchedulerConfiguration.WAIT_TIME_BEFORE_NEXT_STARVATION_CHECK_MS, + 321); + + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig); + + assertTrue("Preemption enabled", + yarnConvertedConfig.getBoolean( + YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, + false)); + assertEquals("Wait time before kill", 123, + yarnConvertedConfig.getInt( + CapacitySchedulerConfiguration.PREEMPTION_WAIT_TIME_BEFORE_KILL, + -1)); + assertEquals("Starvation check wait time", 321, + yarnConvertedConfig.getInt( + CapacitySchedulerConfiguration.PREEMPTION_MONITORING_INTERVAL, + -1)); + } + + @Test + public void testSiteAssignMultipleConversion() { + yarnConfig.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, true); + + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig); + + assertTrue("Assign multiple", + yarnConvertedConfig.getBoolean( + CapacitySchedulerConfiguration.ASSIGN_MULTIPLE_ENABLED, + false)); + } + + @Test + public void testSiteMaxAssignConversion() { + yarnConfig.setInt(FairSchedulerConfiguration.MAX_ASSIGN, 111); + + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig); + + assertEquals("Max assign", 111, + yarnConvertedConfig.getInt( + CapacitySchedulerConfiguration.MAX_ASSIGN_PER_HEARTBEAT, -1)); + } + + @Test + public void testSiteLocalityThresholdConversion() { + yarnConfig.set(FairSchedulerConfiguration.LOCALITY_THRESHOLD_NODE, + "123.123"); + yarnConfig.set(FairSchedulerConfiguration.LOCALITY_THRESHOLD_RACK, + "321.321"); + + converter.convertSiteProperties(yarnConfig, yarnConvertedConfig); + + assertEquals("Locality threshold node", "123.123", + yarnConvertedConfig.get( + CapacitySchedulerConfiguration.NODE_LOCALITY_DELAY)); + assertEquals("Locality threshold rack", "321.321", + yarnConvertedConfig.get( + CapacitySchedulerConfiguration.RACK_LOCALITY_ADDITIONAL_DELAY)); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/conversion-rules.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/conversion-rules.properties new file mode 100644 index 0000000000..b3e421d8d7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/conversion-rules.properties @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License" you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +maxCapacityPercentage.action=abort +maxChildCapacity.action=ABORT +userMaxRunningApps.action=abort +userMaxAppsDefault.action=ABORT +dynamicMaxAssign.action=abort +specifiedNotFirstRule.action=ABORT +reservationSystem.action=abort +queueAutoCreate.action=ABORT \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-conversion.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-conversion.xml new file mode 100644 index 0000000000..67f9ed9a90 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-conversion.xml @@ -0,0 +1,94 @@ + + + + + 1.0 + drf + alice,bob,joe,john hadoop_users + alice,bob,joe,john hadoop_users + + 1.0 + drf + + + 1.0 + drf + + 1.0 + drf + john + john + vcores=2,memory-mb=8192 + + + memory-mb=50.0%, vcores=50.0% + 3.0 + false + drf + joe + joe + + + + memory-mb=8192, vcores=1 + 1.0 + drf + vcores=3,memory-mb=4096 + + memory-mb=16384, vcores=4 + 2 + 3.0 + false + drf + alice + alice + 0.15 + memory-mb=16384, vcores=4 + + + memory-mb=8192, vcores=2 + 1.0 + drf + bob + bob + -1.0 + + + + + 30 + + 10 + 23 + 24 + 0.12 + 15 + fair + 0.16 + + + + + + + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-invalid.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-invalid.xml new file mode 100644 index 0000000000..4afed0e8e4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-invalid.xml @@ -0,0 +1,21 @@ + + + + + \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-max-resources-percentage.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-max-resources-percentage.xml new file mode 100644 index 0000000000..a286ac89c6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-max-resources-percentage.xml @@ -0,0 +1,90 @@ + + + + + 1.0 + drf + alice,bob,joe,john hadoop_users + alice,bob,joe,john hadoop_users + + 1.0 + drf + + + 1.0 + drf + + 1.0 + drf + john + john + vcores=2,memory-mb=8192 + + + memory-mb=50.0%, vcores=50.0% + 3.0 + false + drf + joe + joe + + + + memory-mb=8192, vcores=1 + 1.0 + drf + vcores=3,memory-mb=4096 + + memory-mb=16384, vcores=4 + 2 + 3.0 + false + drf + alice + alice + 0.15 + memory-mb=16384, vcores=4 + + + memory-mb=8192, vcores=2 + 1.0 + drf + bob + bob + -1.0 + + + + 23 + 24 + 0.12 + 15 + fair + 0.16 + + + + + + + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-orderingpolicy-mixed.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-orderingpolicy-mixed.xml new file mode 100644 index 0000000000..3a2a59300c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-orderingpolicy-mixed.xml @@ -0,0 +1,89 @@ + + + + + 1.0 + drf + alice,bob,joe,john hadoop_users + alice,bob,joe,john hadoop_users + + 1.0 + fair + + + 1.0 + drf + + 1.0 + drf + john + john + vcores=2,memory-mb=8192 + + + memory-mb=50.0%, vcores=50.0% + 3.0 + false + drf + joe + joe + + + + memory-mb=8192, vcores=1 + 1.0 + fair + vcores=3,memory-mb=4096 + + memory-mb=16384, vcores=4 + 2 + 3.0 + false + drf + alice + alice + 0.15 + + + memory-mb=8192, vcores=2 + 1.0 + drf + bob + bob + -1.0 + + + + 23 + 24 + 0.12 + 15 + drf + 0.16 + + + + + + + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-orderingpolicy.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-orderingpolicy.xml new file mode 100644 index 0000000000..1b619c2160 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-orderingpolicy.xml @@ -0,0 +1,89 @@ + + + + + 1.0 + fair + alice,bob,joe,john hadoop_users + alice,bob,joe,john hadoop_users + + 1.0 + fair + + + 1.0 + fair + + 1.0 + fifo + john + john + vcores=2,memory-mb=8192 + + + memory-mb=50.0%, vcores=50.0% + 3.0 + false + fair + joe + joe + + + + memory-mb=8192, vcores=1 + 1.0 + fair + vcores=3,memory-mb=4096 + + memory-mb=16384, vcores=4 + 2 + 3.0 + false + fifo + alice + alice + 0.15 + + + memory-mb=8192, vcores=2 + 1.0 + fair + bob + bob + -1.0 + + + + 23 + 24 + 0.12 + 15 + fair + 0.16 + + + + + + + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-sameleafqueue.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-sameleafqueue.xml new file mode 100644 index 0000000000..d28e7b9194 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/fair-scheduler-sameleafqueue.xml @@ -0,0 +1,90 @@ + + + + + 1.0 + drf + alice,bob,joe,john hadoop_users + alice,bob,joe,john hadoop_users + + 1.0 + drf + + + 1.0 + drf + + 1.0 + drf + john + john + vcores=2,memory-mb=8192 + + + memory-mb=50.0%, vcores=50.0% + 3.0 + false + drf + joe + joe + + + + memory-mb=8192, vcores=1 + 1.0 + drf + vcores=3,memory-mb=4096 + + memory-mb=16384, vcores=4 + 2 + 3.0 + false + drf + alice + alice + 0.15 + memory-mb=16384, vcores=4 + + + memory-mb=8192, vcores=2 + 1.0 + drf + bob + bob + -1.0 + + + + 23 + 24 + 0.12 + 15 + fair + 0.16 + + + + + + + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/yarn-site-with-allocation-file-ref.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/yarn-site-with-allocation-file-ref.xml new file mode 100644 index 0000000000..5352d43451 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/yarn-site-with-allocation-file-ref.xml @@ -0,0 +1,23 @@ + + + + + + yarn.scheduler.fair.allocation.file + fair-scheduler-conversion.xml + + \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/yarn-site-with-invalid-allocation-file-ref.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/yarn-site-with-invalid-allocation-file-ref.xml new file mode 100644 index 0000000000..248040d928 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/yarn-site-with-invalid-allocation-file-ref.xml @@ -0,0 +1,37 @@ + + + + + + + + + yarn.scheduler.fair.allocation.file + + \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md index e17538ccdf..c0bffa29ff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md @@ -193,6 +193,7 @@ Usage: `yarn resourcemanager [-format-state-store]` |:---- |:---- | | -format-state-store | Formats the RMStateStore. This will clear the RMStateStore and is useful if past applications are no longer needed. This should be run only when the ResourceManager is not running. | | -remove-application-from-state-store \ | Remove the application from RMStateStore. This should be run only when the ResourceManager is not running. | +| -convert-fs-configuration [-y|yarnsiteconfig] [-f|fsconfig] [-r|rulesconfig] [-o|output-directory] [-p|print] [-c|cluster-resource] | WARNING: This feature is experimental and not intended for production use! Development is still in progress so the converter should not be considered complete!
Converts the specified Fair Scheduler configuration to Capacity Scheduler configuration. Requires two mandatory input files. First, the yarn-site.xml with the following format: [-y|yarnsiteconfig [\]. Secondly, the fair-scheduler.xml with the following format: [-f|fsconfig [\]. This config is not mandatory if there is a reference in yarn-site.xml to the fair-scheduler.xml with the property 'yarn.scheduler.fair.allocation.file'. If both are defined, the -f option has precedence. The output directory of the config files should be specified as well, with: \[-o|output-directory\ \]. An optional rules config file could be also specified with the following format: [-r|rulesconfig \]. The rule config file's format is a property file. There's an additional \[-p|print\] parameter, which is optional. If defined, the configuration will be emitted to the console instead. In its normal operation, the output files (yarn-site.xml and capacity-scheduler.xml) of this command is generated to the specified output directory. The cluster resource parameter (\[-c|cluster-resource\] \\]) needs to be specified if any queue has a maxResources setting with value as percentage. The format of the resource string is the same as in fair-scheduler.xml.) ] | Start the ResourceManager