big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-5470. LocalJobRunner does not work on Windows. Contributed by Sandy Ryza.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1516623 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Chris Nauroth 2013-08-22 21:39:30 +00:00
parent d1dfa5f923
commit 200220e8f3
2 changed files with 13 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -234,6 +234,9 @@ Release 2.1.1-beta - UNRELEASED
MAPREDUCE-5468. Fix MR AM recovery for map-only jobs. (vinodkv via
acmurthy)
MAPREDUCE-5470. LocalJobRunner does not work on Windows. (Sandy Ryza via
cnauroth)
Release 2.1.0-beta - 2013-08-22
INCOMPATIBLE CHANGES

17
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
View File

@ -227,7 +227,7 @@ public void run() {
info.getSplitIndex(), 1);
map.setUser(UserGroupInformation.getCurrentUser().
getShortUserName());
setupChildMapredLocalDirs(localJobDir, map, localConf);
setupChildMapredLocalDirs(map, localConf);
MapOutputFile mapOutput = new MROutputFiles();
mapOutput.setConf(localConf);
@ -305,7 +305,7 @@ public void run() {
reduceId, taskId, mapIds.size(), 1);
reduce.setUser(UserGroupInformation.getCurrentUser().
getShortUserName());
setupChildMapredLocalDirs(localJobDir, reduce, localConf);
setupChildMapredLocalDirs(reduce, localConf);
reduce.setLocalMapFiles(mapOutputFiles);
if (!Job.this.isInterrupted()) {
@ -958,16 +958,18 @@ public LogParams getLogFileParams(org.apache.hadoop.mapreduce.JobID jobID,
throw new UnsupportedOperationException("Not supported");
}
static void setupChildMapredLocalDirs(Path localJobDir, Task t, JobConf conf) {
static void setupChildMapredLocalDirs(Task t, JobConf conf) {
String[] localDirs = conf.getTrimmedStrings(MRConfig.LOCAL_DIR);
String jobId = t.getJobID().toString();
String taskId = t.getTaskID().toString();
boolean isCleanup = t.isTaskCleanupTask();
String user = t.getUser();
StringBuffer childMapredLocalDir =
new StringBuffer(localDirs[0] + Path.SEPARATOR
+ getLocalTaskDir(localJobDir, taskId, isCleanup));
+ getLocalTaskDir(user, jobId, taskId, isCleanup));
for (int i = 1; i < localDirs.length; i++) {
childMapredLocalDir.append("," + localDirs[i] + Path.SEPARATOR
+ getLocalTaskDir(localJobDir, taskId, isCleanup));
+ getLocalTaskDir(user, jobId, taskId, isCleanup));
}
LOG.debug(MRConfig.LOCAL_DIR + " for child : " + childMapredLocalDir);
conf.set(MRConfig.LOCAL_DIR, childMapredLocalDir.toString());
@ -976,9 +978,10 @@ static void setupChildMapredLocalDirs(Path localJobDir, Task t, JobConf conf) {
static final String TASK_CLEANUP_SUFFIX = ".cleanup";
static final String JOBCACHE = "jobcache";
static String getLocalTaskDir(Path localJobDir, String taskid,
static String getLocalTaskDir(String user, String jobid, String taskid,
boolean isCleanupAttempt) {
String taskDir = localJobDir.toString() + Path.SEPARATOR + taskid;
String taskDir = jobDir + Path.SEPARATOR + user + Path.SEPARATOR + JOBCACHE
+ Path.SEPARATOR + jobid + Path.SEPARATOR + taskid;
if (isCleanupAttempt) {
taskDir = taskDir + TASK_CLEANUP_SUFFIX;
}