big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-8798. [Submarine] Job should not be submitted if --input_path option is missing. (Zhankun Tang via wangda)

Browse Source 
Change-Id: I7ae0e44eb5179b04a6ac861ec1c65f3b18c38f0f
This commit is contained in:
Wangda Tan 2018-10-16 13:39:34 -07:00
parent 46d6e00166
commit 143d74775b
2 changed files with 45 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java
View File

@ -62,6 +62,12 @@ public void updateParametersByParsedCommandline(CommandLine parsedCommandLine,
if (parsedCommandLine.getOptionValue(CliConstants.N_WORKERS) != null) {
nWorkers = Integer.parseInt(
parsedCommandLine.getOptionValue(CliConstants.N_WORKERS));
// Only check null value.
// Training job shouldn't ignore INPUT_PATH option
// But if nWorkers is 0, INPUT_PATH can be ignored because user can only run Tensorboard
if (null == input && 0 != nWorkers) {
throw new ParseException("\"--" + CliConstants.INPUT_PATH + "\" is absent");
}
}
int nPS = 0;

39
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/TestRunJobCliParsing.java
View File

@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.submarine.client.cli;
import org.apache.commons.cli.ParseException;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
@ -135,6 +136,44 @@ public void testBasicRunJobForSingleNodeTraining() throws Exception {
Assert.assertTrue(jobRunParameters.isWaitJobFinish());
}
@Test
public void testNoInputPathOptionSpecified() throws Exception {
RunJobCli runJobCli = new RunJobCli(getMockClientContext());
String expectedErrorMessage = "\"--" + CliConstants.INPUT_PATH + "\" is absent";
String actualMessage = "";
try {
runJobCli.run(
new String[]{"--name", "my-job", "--docker_image", "tf-docker:1.1.0",
"--checkpoint_path", "hdfs://output",
"--num_workers", "1", "--worker_launch_cmd", "python run-job.py",
"--worker_resources", "memory=4g,vcores=2", "--tensorboard",
"true", "--verbose", "--wait_job_finish"});
} catch (ParseException e) {
actualMessage = e.getMessage();
e.printStackTrace();
}
Assert.assertEquals(expectedErrorMessage, actualMessage);
}
/**
* when only run tensorboard, input_path is not needed
* */
@Test
public void testNoInputPathOptionButOnlyRunTensorboard() throws Exception {
RunJobCli runJobCli = new RunJobCli(getMockClientContext());
boolean success = true;
try {
runJobCli.run(
new String[]{"--name", "my-job", "--docker_image", "tf-docker:1.1.0",
"--num_workers", "0", "--tensorboard", "--verbose",
"--tensorboard_resources", "memory=2G,vcores=2",
"--tensorboard_docker_image", "tb_docker_image:001"});
} catch (ParseException e) {
success = false;
}
Assert.assertTrue(success);
}
@Test
public void testLaunchCommandPatternReplace() throws Exception {
RunJobCli runJobCli = new RunJobCli(getMockClientContext());