From 200220e8f3684258c281736fd31f0ebdcbac91f0 Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Thu, 22 Aug 2013 21:39:30 +0000 Subject: [PATCH] MAPREDUCE-5470. LocalJobRunner does not work on Windows. Contributed by Sandy Ryza. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1516623 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../apache/hadoop/mapred/LocalJobRunner.java | 17 ++++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index d0345594ac..85650bd504 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -234,6 +234,9 @@ Release 2.1.1-beta - UNRELEASED MAPREDUCE-5468. Fix MR AM recovery for map-only jobs. (vinodkv via acmurthy) + MAPREDUCE-5470. LocalJobRunner does not work on Windows. (Sandy Ryza via + cnauroth) + Release 2.1.0-beta - 2013-08-22 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java index 95c272d3d2..2bb7dc8365 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java @@ -227,7 +227,7 @@ public void run() { info.getSplitIndex(), 1); map.setUser(UserGroupInformation.getCurrentUser(). getShortUserName()); - setupChildMapredLocalDirs(localJobDir, map, localConf); + setupChildMapredLocalDirs(map, localConf); MapOutputFile mapOutput = new MROutputFiles(); mapOutput.setConf(localConf); @@ -305,7 +305,7 @@ public void run() { reduceId, taskId, mapIds.size(), 1); reduce.setUser(UserGroupInformation.getCurrentUser(). getShortUserName()); - setupChildMapredLocalDirs(localJobDir, reduce, localConf); + setupChildMapredLocalDirs(reduce, localConf); reduce.setLocalMapFiles(mapOutputFiles); if (!Job.this.isInterrupted()) { @@ -958,16 +958,18 @@ public LogParams getLogFileParams(org.apache.hadoop.mapreduce.JobID jobID, throw new UnsupportedOperationException("Not supported"); } - static void setupChildMapredLocalDirs(Path localJobDir, Task t, JobConf conf) { + static void setupChildMapredLocalDirs(Task t, JobConf conf) { String[] localDirs = conf.getTrimmedStrings(MRConfig.LOCAL_DIR); + String jobId = t.getJobID().toString(); String taskId = t.getTaskID().toString(); boolean isCleanup = t.isTaskCleanupTask(); + String user = t.getUser(); StringBuffer childMapredLocalDir = new StringBuffer(localDirs[0] + Path.SEPARATOR - + getLocalTaskDir(localJobDir, taskId, isCleanup)); + + getLocalTaskDir(user, jobId, taskId, isCleanup)); for (int i = 1; i < localDirs.length; i++) { childMapredLocalDir.append("," + localDirs[i] + Path.SEPARATOR - + getLocalTaskDir(localJobDir, taskId, isCleanup)); + + getLocalTaskDir(user, jobId, taskId, isCleanup)); } LOG.debug(MRConfig.LOCAL_DIR + " for child : " + childMapredLocalDir); conf.set(MRConfig.LOCAL_DIR, childMapredLocalDir.toString()); @@ -976,9 +978,10 @@ static void setupChildMapredLocalDirs(Path localJobDir, Task t, JobConf conf) { static final String TASK_CLEANUP_SUFFIX = ".cleanup"; static final String JOBCACHE = "jobcache"; - static String getLocalTaskDir(Path localJobDir, String taskid, + static String getLocalTaskDir(String user, String jobid, String taskid, boolean isCleanupAttempt) { - String taskDir = localJobDir.toString() + Path.SEPARATOR + taskid; + String taskDir = jobDir + Path.SEPARATOR + user + Path.SEPARATOR + JOBCACHE + + Path.SEPARATOR + jobid + Path.SEPARATOR + taskid; if (isCleanupAttempt) { taskDir = taskDir + TASK_CLEANUP_SUFFIX; }