big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10175. FS-CS converter: only convert placement rules if a cmd line switch is defined. Contributed by Peter Bacsko

Browse Source
This commit is contained in:
Szilard Nemeth 2020-03-03 09:46:32 +01:00
parent edc2e9d2f1
commit c0d084247c
5 changed files with 131 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigArgumentHandler.java
View File

@ -98,6 +98,10 @@ public enum CliOption {
"Disables checking whether a placement rule is terminal to maintain" +
" backward compatibility with configs that were made before YARN-8967.",
false),
CONVERT_PLACEMENT_RULES("convert placement rules",
"m", "convert-placement-rules",
"Convert Fair Scheduler placement rules to Capacity" +
" Scheduler mapping rules", false),
HELP("help", "h", "help", "Displays the list of options", false);
private final String name;
@ -216,6 +220,8 @@ private FSConfigToCSConfigConverterParams validateInputFiles(
cliParser.getOptionValue(CliOption.CONVERSION_RULES.shortSwitch);
String outputDir =
cliParser.getOptionValue(CliOption.OUTPUT_DIR.shortSwitch);
boolean convertPlacementRules =
cliParser.hasOption(CliOption.CONVERT_PLACEMENT_RULES.shortSwitch);
checkFile(CliOption.YARN_SITE, yarnSiteXmlFile);
checkFile(CliOption.FAIR_SCHEDULER, fairSchedulerXmlFile);
@ -231,6 +237,7 @@ private FSConfigToCSConfigConverterParams validateInputFiles(
cliParser.getOptionValue(CliOption.CLUSTER_RESOURCE.shortSwitch))
.withConsole(cliParser.hasOption(CliOption.CONSOLE_MODE.shortSwitch))
.withOutputDirectory(outputDir)
.withConvertPlacementRules(convertPlacementRules)
.build();
}

35
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java
View File

@ -91,10 +91,12 @@ public class FSConfigToCSConfigConverter {
private Configuration convertedYarnSiteConfig;
private Configuration capacitySchedulerConfig;
private FSConfigToCSConfigRuleHandler ruleHandler;
private QueuePlacementConverter placementConverter;
private OutputStream yarnSiteOutputStream;
private OutputStream capacitySchedulerOutputStream;
private boolean consoleMode = false;
private boolean convertPlacementRules = false;
public FSConfigToCSConfigConverter(FSConfigToCSConfigRuleHandler
ruleHandler, ConversionOptions conversionOptions) {
@ -102,6 +104,7 @@ public FSConfigToCSConfigConverter(FSConfigToCSConfigRuleHandler
this.conversionOptions = conversionOptions;
this.yarnSiteOutputStream = System.out;
this.capacitySchedulerOutputStream = System.out;
this.placementConverter = new QueuePlacementConverter();
}
public void convert(FSConfigToCSConfigConverterParams params)
@ -113,6 +116,8 @@ public void convert(FSConfigToCSConfigConverterParams params)
handleFairSchedulerConfig(params, inputYarnSiteConfig);
this.clusterResource = getClusterResource(params);
this.convertPlacementRules = params.isConvertPlacementRules();
convert(inputYarnSiteConfig);
}
@ -309,16 +314,19 @@ private void convertCapacitySchedulerXml(FairScheduler fs) {
queueConverter.convertQueueHierarchy(rootQueue);
emitACLs(fs);
PlacementManager placementManager =
fs.getRMContext().getQueuePlacementManager();
if (convertPlacementRules) {
LOG.info("Converting placement rules");
PlacementManager placementManager =
fs.getRMContext().getQueuePlacementManager();
if (placementManager.getPlacementRules().size() > 0) {
QueuePlacementConverter placementConverter =
new QueuePlacementConverter();
Map<String, String> properties =
placementConverter.convertPlacementPolicy(placementManager,
ruleHandler, userAsDefaultQueue);
properties.forEach((k, v) -> capacitySchedulerConfig.set(k, v));
if (placementManager.getPlacementRules().size() > 0) {
Map<String, String> properties =
placementConverter.convertPlacementPolicy(placementManager,
ruleHandler, userAsDefaultQueue);
properties.forEach((k, v) -> capacitySchedulerConfig.set(k, v));
}
} else {
LOG.info("Ignoring the conversion of placement rules");
}
}
@ -432,6 +440,15 @@ Configuration getYarnSiteConfig() {
return convertedYarnSiteConfig;
}
@VisibleForTesting
void setConvertPlacementRules(boolean convertPlacementRules) {
this.convertPlacementRules = convertPlacementRules;
}
@VisibleForTesting
void setPlacementConverter(QueuePlacementConverter converter) {
this.placementConverter = converter;
}
/*
* Determines whether <queuePlacementPolicy> is present
* in the allocation file or not.

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverterParams.java
View File

@ -27,6 +27,9 @@ public final class FSConfigToCSConfigConverterParams {
private boolean console;
private String clusterResource;
private String outputDirectory;
private boolean convertPlacementRules;
private FSConfigToCSConfigConverterParams() {
//must use builder
@ -56,6 +59,10 @@ public String getOutputDirectory() {
return outputDirectory;
}
public boolean isConvertPlacementRules() {
return convertPlacementRules;
}
@Override
public String toString() {
return "FSConfigToCSConfigConverterParams{" +
@ -63,7 +70,8 @@ public String toString() {
", fairSchedulerXmlConfig='" + fairSchedulerXmlConfig + '\'' +
", conversionRulesConfig='" + conversionRulesConfig + '\'' +
", clusterResource='" + clusterResource + '\'' +
", console=" + console +
", console=" + console + '\'' +
", convertPlacementRules=" + convertPlacementRules +
'}';
}
@ -78,6 +86,7 @@ public static final class Builder {
private boolean console;
private String clusterResource;
private String outputDirectory;
private boolean convertPlacementRules;
private Builder() {
}
@ -116,6 +125,11 @@ public Builder withOutputDirectory(String outputDir) {
return this;
}
public Builder withConvertPlacementRules(boolean convertPlacementRules) {
this.convertPlacementRules = convertPlacementRules;
return this;
}
public FSConfigToCSConfigConverterParams build() {
FSConfigToCSConfigConverterParams params =
new FSConfigToCSConfigConverterParams();
@ -125,6 +139,7 @@ public FSConfigToCSConfigConverterParams build() {
params.yarnSiteXmlConfig = this.yarnSiteXmlConfig;
params.conversionRulesConfig = this.conversionRulesConfig;
params.outputDirectory = this.outputDirectory;
params.convertPlacementRules = this.convertPlacementRules;
return params;
}
}

43
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java
View File

@ -19,6 +19,7 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.verify;
import java.io.File;
import java.io.IOException;
@ -555,4 +556,46 @@ public void testCapacitySchedulerXmlExistsInOutputFolder()
testFileExistsInOutputFolder(
YarnConfiguration.CS_CONFIGURATION_FILE);
}
@Test
public void testPlacementRulesConversionEnabled() throws Exception {
testPlacementRuleConversion(true);
}
@Test
public void testPlacementRulesConversionDisabled() throws Exception {
testPlacementRuleConversion(false);
}
private void testPlacementRuleConversion(boolean enabled) throws Exception {
setupFSConfigConversionFiles(true);
String[] args = null;
if (enabled) {
args = getArgumentsAsArrayWithDefaults("-f",
FSConfigConverterTestCommons.FS_ALLOC_FILE,
"-p", "-m");
} else {
args = getArgumentsAsArrayWithDefaults("-f",
FSConfigConverterTestCommons.FS_ALLOC_FILE,
"-p");
}
FSConfigToCSConfigArgumentHandler argumentHandler =
new FSConfigToCSConfigArgumentHandler(conversionOptions);
argumentHandler.setConverterSupplier(this::getMockConverter);
argumentHandler.parseAndConvert(args);
ArgumentCaptor<FSConfigToCSConfigConverterParams> captor =
ArgumentCaptor.forClass(FSConfigToCSConfigConverterParams.class);
verify(mockConverter).convert(captor.capture());
FSConfigToCSConfigConverterParams params = captor.getValue();
if (enabled) {
assertTrue("-m switch had no effect", params.isConvertPlacementRules());
} else {
assertFalse("Placement rule conversion was enabled",
params.isConvertPlacementRules());
}
}
}

39
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java
View File

@ -31,6 +31,9 @@
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyZeroInteractions;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
@ -43,6 +46,7 @@
import org.apache.hadoop.service.ServiceStateException;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
@ -86,6 +90,9 @@ public class TestFSConfigToCSConfigConverter {
@Mock
private DryRunResultHolder dryRunResultHolder;
@Mock
private QueuePlacementConverter placementConverter;
private FSConfigToCSConfigConverter converter;
private Configuration config;
@ -583,6 +590,7 @@ private void testUserAsDefaultQueueWithoutPlacementRules(boolean
config.setBoolean(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE,
userAsDefaultQueue);
converter.setConvertPlacementRules(true);
converter.convert(config);
Configuration convertedConf = getConvertedCSConfig();
@ -650,6 +658,37 @@ private void testAutoCreateChildQueuesWithoutPlacementRules(
}
}
@Test
public void testPlacementRulesConversionDisabled() throws Exception {
FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder()
.withClusterResource(CLUSTER_RESOURCE_STRING)
.withFairSchedulerXmlConfig(FAIR_SCHEDULER_XML)
.withConvertPlacementRules(false)
.build();
converter.setPlacementConverter(placementConverter);
converter.convert(params);
verifyZeroInteractions(placementConverter);
}
@Test
public void testPlacementRulesConversionEnabled() throws Exception {
FSConfigToCSConfigConverterParams params = createDefaultParamsBuilder()
.withClusterResource(CLUSTER_RESOURCE_STRING)
.withFairSchedulerXmlConfig(FAIR_SCHEDULER_XML)
.withConvertPlacementRules(true)
.build();
converter.setPlacementConverter(placementConverter);
converter.convert(params);
verify(placementConverter).convertPlacementPolicy(
any(PlacementManager.class),
any(FSConfigToCSConfigRuleHandler.class),
any(Boolean.class));
}
private Configuration getConvertedCSConfig() {
ByteArrayInputStream input =
new ByteArrayInputStream(csConfigOut.toByteArray());