big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-8817. [Submarine] In cases when user doesn't ask HDFS path while submitting job but framework requires user to set HDFS related environments. Contributed by Wangda Tan.

Browse Source
This commit is contained in:
Sunil G 2018-09-25 11:23:34 +05:30
parent 9c3fbbc4f6
commit d0bc6a27fd
1 changed files with 10 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
View File

@ -102,6 +102,13 @@ private String getValueOfEnvionment(String envar) {
}
}
private boolean needHdfs(String content) {
if (content != null && content.contains("hdfs://")) {
return true;
}
return false;
}
private void addHdfsClassPathIfNeeded(RunJobParameters parameters,
PrintWriter fw, Component comp) throws IOException {
// Find envs to use HDFS
@ -121,14 +128,11 @@ private void addHdfsClassPathIfNeeded(RunJobParameters parameters,
boolean lackingEnvs = false;
if ((parameters.getInputPath() != null && parameters.getInputPath()
.contains("hdfs://")) || (parameters.getCheckpointPath() != null
&& parameters.getCheckpointPath().contains("hdfs://")) || (
parameters.getSavedModelPath() != null && parameters.getSavedModelPath()
.contains("hdfs://")) || hadoopEnv) {
if (needHdfs(parameters.getInputPath()) || needHdfs(
parameters.getPSLaunchCmd()) || needHdfs(
parameters.getWorkerLaunchCmd()) || hadoopEnv) {
// HDFS is asked either in input or output, set LD_LIBRARY_PATH
// and classpath
if (hdfsHome != null) {
// Unset HADOOP_HOME/HADOOP_YARN_HOME to make sure host machine's envs
// won't pollute docker's env.