big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-14817. [Dynamometer] Fix start script options parsing which incorrectly interpret options starting with h as a help argument. Contributed by Soya Miyoshi.

Browse Source
This commit is contained in:
Erik Krogen 2019-09-06 10:24:51 -07:00
parent c92a3e94d8
commit 9637097ef9
1 changed files with 30 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/Client.java
View File

@ -387,15 +387,14 @@ private void printUsage() {
*/ */
public boolean init(String[] args) throws ParseException, IOException { public boolean init(String[] args) throws ParseException, IOException {
CommandLineParser parser = new GnuParser(); List<String> list = Arrays.asList(args);
if (parser.parse( if (list.contains("-h") || list.contains("--help")) {
new Options().addOption("h", "help", false, "Shows this message."),
args, true).hasOption("h")) {
printUsage(); printUsage();
return false; return false;
} }
CommandLine cliParser = parser.parse(opts, args); CommandLineParser parser = new GnuParser();
CommandLine commandLine = parser.parse(opts, args);
yarnClient = YarnClient.createYarnClient(); yarnClient = YarnClient.createYarnClient();
yarnClient.init(getConf()); yarnClient.init(getConf());
@ -403,12 +402,13 @@ public boolean init(String[] args) throws ParseException, IOException {
LOG.info("Starting with arguments: [\"{}\"]", LOG.info("Starting with arguments: [\"{}\"]",
Joiner.on("\" \"").join(args)); Joiner.on("\" \"").join(args));
Path fsImageDir = new Path(cliParser.getOptionValue(FS_IMAGE_DIR_ARG, "")); Path fsImageDir = new Path(commandLine.getOptionValue(FS_IMAGE_DIR_ARG,
""));
versionFilePath = new Path(fsImageDir, "VERSION").toString(); versionFilePath = new Path(fsImageDir, "VERSION").toString();
if (cliParser.hasOption(NAMENODE_SERVICERPC_ADDR_ARG)) { if (commandLine.hasOption(NAMENODE_SERVICERPC_ADDR_ARG)) {
launchNameNode = false; launchNameNode = false;
remoteNameNodeRpcAddress = remoteNameNodeRpcAddress =
cliParser.getOptionValue(NAMENODE_SERVICERPC_ADDR_ARG); commandLine.getOptionValue(NAMENODE_SERVICERPC_ADDR_ARG);
} else { } else {
launchNameNode = true; launchNameNode = true;
FileSystem localFS = FileSystem.getLocal(getConf()); FileSystem localFS = FileSystem.getLocal(getConf());
@ -434,26 +434,27 @@ public boolean init(String[] args) throws ParseException, IOException {
+ "application master, exiting. Specified virtual cores=" + amVCores); + "application master, exiting. Specified virtual cores=" + amVCores);
} }
this.appName = cliParser.getOptionValue(APPNAME_ARG, APPNAME_DEFAULT); this.appName = commandLine.getOptionValue(APPNAME_ARG, APPNAME_DEFAULT);
this.amQueue = cliParser.getOptionValue(QUEUE_ARG, QUEUE_DEFAULT); this.amQueue = commandLine.getOptionValue(QUEUE_ARG, QUEUE_DEFAULT);
this.amMemory = Integer.parseInt(cliParser this.amMemory = Integer.parseInt(commandLine
.getOptionValue(MASTER_MEMORY_MB_ARG, MASTER_MEMORY_MB_DEFAULT)); .getOptionValue(MASTER_MEMORY_MB_ARG, MASTER_MEMORY_MB_DEFAULT));
this.amVCores = Integer.parseInt( this.amVCores = Integer.parseInt(
cliParser.getOptionValue(MASTER_VCORES_ARG, MASTER_VCORES_DEFAULT)); commandLine.getOptionValue(MASTER_VCORES_ARG, MASTER_VCORES_DEFAULT));
this.confPath = cliParser.getOptionValue(CONF_PATH_ARG); this.confPath = commandLine.getOptionValue(CONF_PATH_ARG);
this.blockListPath = cliParser.getOptionValue(BLOCK_LIST_PATH_ARG); this.blockListPath = commandLine.getOptionValue(BLOCK_LIST_PATH_ARG);
if (cliParser.hasOption(HADOOP_BINARY_PATH_ARG)) { if (commandLine.hasOption(HADOOP_BINARY_PATH_ARG)) {
this.hadoopBinary = cliParser.getOptionValue(HADOOP_BINARY_PATH_ARG); this.hadoopBinary = commandLine.getOptionValue(HADOOP_BINARY_PATH_ARG);
} else { } else {
this.hadoopBinary = DynoInfraUtils.fetchHadoopTarball( this.hadoopBinary = DynoInfraUtils.fetchHadoopTarball(
new File(".").getAbsoluteFile(), new File(".").getAbsoluteFile(),
cliParser.getOptionValue(HADOOP_VERSION_ARG), getConf(), LOG) commandLine.getOptionValue(HADOOP_VERSION_ARG), getConf(), LOG)
.toString(); .toString();
} }
this.amOptions = AMOptions.initFromParser(cliParser); this.amOptions = AMOptions.initFromParser(commandLine);
this.clientTimeout = Integer this.clientTimeout = Integer
.parseInt(cliParser.getOptionValue(TIMEOUT_ARG, TIMEOUT_DEFAULT)); .parseInt(commandLine.getOptionValue(TIMEOUT_ARG, TIMEOUT_DEFAULT));
this.tokenFileLocation = cliParser.getOptionValue(TOKEN_FILE_LOCATION_ARG); this.tokenFileLocation = commandLine.
getOptionValue(TOKEN_FILE_LOCATION_ARG);
amOptions.verify(); amOptions.verify();
@ -467,28 +468,28 @@ public boolean init(String[] args) throws ParseException, IOException {
numTotalDataNodes = blockListFS.listStatus(blockPath, numTotalDataNodes = blockListFS.listStatus(blockPath,
DynoConstants.BLOCK_LIST_FILE_FILTER).length; DynoConstants.BLOCK_LIST_FILE_FILTER).length;
if (cliParser.hasOption(WORKLOAD_REPLAY_ENABLE_ARG)) { if (commandLine.hasOption(WORKLOAD_REPLAY_ENABLE_ARG)) {
if (!cliParser.hasOption(WORKLOAD_INPUT_PATH_ARG) if (!commandLine.hasOption(WORKLOAD_INPUT_PATH_ARG)
|| !cliParser.hasOption(WORKLOAD_START_DELAY_ARG)) { || !commandLine.hasOption(WORKLOAD_START_DELAY_ARG)) {
throw new IllegalArgumentException("workload_replay_enable was " throw new IllegalArgumentException("workload_replay_enable was "
+ "specified; must include all required workload_ parameters."); + "specified; must include all required workload_ parameters.");
} }
launchWorkloadJob = true; launchWorkloadJob = true;
workloadInputPath = cliParser.getOptionValue(WORKLOAD_INPUT_PATH_ARG); workloadInputPath = commandLine.getOptionValue(WORKLOAD_INPUT_PATH_ARG);
workloadThreadsPerMapper = Integer workloadThreadsPerMapper = Integer
.parseInt(cliParser.getOptionValue(WORKLOAD_THREADS_PER_MAPPER_ARG, .parseInt(commandLine.getOptionValue(WORKLOAD_THREADS_PER_MAPPER_ARG,
String.valueOf(AuditReplayMapper.NUM_THREADS_DEFAULT))); String.valueOf(AuditReplayMapper.NUM_THREADS_DEFAULT)));
workloadRateFactor = Double.parseDouble(cliParser.getOptionValue( workloadRateFactor = Double.parseDouble(commandLine.getOptionValue(
WORKLOAD_RATE_FACTOR_ARG, WORKLOAD_RATE_FACTOR_DEFAULT)); WORKLOAD_RATE_FACTOR_ARG, WORKLOAD_RATE_FACTOR_DEFAULT));
workloadExtraConfigs = new HashMap<>(); workloadExtraConfigs = new HashMap<>();
if (cliParser.getOptionValues(WORKLOAD_CONFIG_ARG) != null) { if (commandLine.getOptionValues(WORKLOAD_CONFIG_ARG) != null) {
for (String opt : cliParser.getOptionValues(WORKLOAD_CONFIG_ARG)) { for (String opt : commandLine.getOptionValues(WORKLOAD_CONFIG_ARG)) {
Iterator<String> kvPair = Iterator<String> kvPair =
Splitter.on("=").trimResults().split(opt).iterator(); Splitter.on("=").trimResults().split(opt).iterator();
workloadExtraConfigs.put(kvPair.next(), kvPair.next()); workloadExtraConfigs.put(kvPair.next(), kvPair.next());
} }
} }
String delayString = cliParser.getOptionValue(WORKLOAD_START_DELAY_ARG, String delayString = commandLine.getOptionValue(WORKLOAD_START_DELAY_ARG,
WorkloadDriver.START_TIME_OFFSET_DEFAULT); WorkloadDriver.START_TIME_OFFSET_DEFAULT);
// Store a temporary config to leverage Configuration's time duration // Store a temporary config to leverage Configuration's time duration
// parsing. // parsing.