big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-1926. Changed DistributedShell to use appIDs as unique identifiers for HDFS paths and thus avoid test failures on Windows. Contributed by Varun Vasudev.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1586562 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2014-04-11 04:06:40 +00:00
parent 8d41b363b8
commit d6a6e982a8
3 changed files with 11 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-yarn-project/CHANGES.txt
View File

@ -111,6 +111,10 @@ Release 2.4.1 - UNRELEASED
when Application(Attempt) state hasn't been stored before. (Jian He via
zjshen)
YARN-1926. Changed DistributedShell to use appIDs as unique identifiers for
HDFS paths and thus avoid test failures on Windows. (Varun Vasudev via
vinodkv)
Release 2.4.0 - 2014-04-07
INCOMPATIBLE CHANGES

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
View File

@ -468,12 +468,12 @@ public boolean run() throws IOException, YarnException {
// Copy the application master jar to the filesystem
// Create a local resource to point to the destination jar path
FileSystem fs = FileSystem.get(conf);
addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.getId(),
addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.toString(),
localResources, null);
// Set the log4j properties if needed
if (!log4jPropFile.isEmpty()) {
addToLocalResources(fs, log4jPropFile, log4jPath, appId.getId(),
addToLocalResources(fs, log4jPropFile, log4jPath, appId.toString(),
localResources, null);
}
@ -489,7 +489,7 @@ public boolean run() throws IOException, YarnException {
if (!shellScriptPath.isEmpty()) {
Path shellSrc = new Path(shellScriptPath);
String shellPathSuffix =
appName + "/" + appId.getId() + "/" + SCRIPT_PATH;
appName + "/" + appId.toString() + "/" + SCRIPT_PATH;
Path shellDst =
new Path(fs.getHomeDirectory(), shellPathSuffix);
fs.copyFromLocalFile(false, true, shellSrc, shellDst);
@ -500,12 +500,12 @@ public boolean run() throws IOException, YarnException {
}
if (!shellCommand.isEmpty()) {
addToLocalResources(fs, null, shellCommandPath, appId.getId(),
addToLocalResources(fs, null, shellCommandPath, appId.toString(),
localResources, shellCommand);
}
if (shellArgs.length > 0) {
addToLocalResources(fs, null, shellArgsPath, appId.getId(),
addToLocalResources(fs, null, shellArgsPath, appId.toString(),
localResources, StringUtils.join(shellArgs, " "));
}
// Set local resource info into app master container launch context
@ -737,7 +737,7 @@ private void forceKillApplication(ApplicationId appId)
}
private void addToLocalResources(FileSystem fs, String fileSrcPath,
String fileDstPath, int appId, Map<String, LocalResource> localResources,
String fileDstPath, String appId, Map<String, LocalResource> localResources,
String resources) throws IOException {
String suffix =
appName + "/" + appId + "/" + fileDstPath;

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
View File

@ -219,7 +219,7 @@ public void testDSRestartWithPreviousRunningContainers() throws Exception {
"--num_containers",
"1",
"--shell_command",
Shell.WINDOWS ? "timeout 8" : "sleep 8",
"sleep 8",
"--master_memory",
"512",
"--container_memory",