From 0f1f5491bc4c6ce9e56c178430a7bb05e2b25843 Mon Sep 17 00:00:00 2001 From: Thomas White Date: Tue, 8 Jan 2013 16:26:33 +0000 Subject: [PATCH] MAPREDUCE-4278. Cannot run two local jobs in parallel from the same gateway. Contributed by Sandy Ryza. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1430363 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/mapred/LocalJobRunner.java | 11 ++++++++--- .../main/java/org/apache/hadoop/mapreduce/JobID.java | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 49e4c0ed0b..5d342374be 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -238,6 +238,9 @@ Release 2.0.3-alpha - Unreleased MAPREDUCE-4856. TestJobOutputCommitter uses same directory as TestJobCleanup. (Sandy Ryza via tomwhite) + MAPREDUCE-4278. Cannot run two local jobs in parallel from the same + gateway. (Sandy Ryza via tomwhite) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java index 6b876356ca..85f8c183fa 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java @@ -609,8 +609,12 @@ public LocalJobRunner(JobConf conf) throws IOException { // JobSubmissionProtocol methods private static int jobid = 0; + // used for making sure that local jobs run in different jvms don't + // collide on staging or job directories + private int randid; + public synchronized org.apache.hadoop.mapreduce.JobID getNewJobID() { - return new org.apache.hadoop.mapreduce.JobID("local", ++jobid); + return new org.apache.hadoop.mapreduce.JobID("local" + randid, ++jobid); } public org.apache.hadoop.mapreduce.JobStatus submitJob( @@ -739,10 +743,11 @@ public String getStagingAreaDir() throws IOException { "/tmp/hadoop/mapred/staging")); UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); String user; + randid = rand.nextInt(Integer.MAX_VALUE); if (ugi != null) { - user = ugi.getShortUserName() + rand.nextInt(); + user = ugi.getShortUserName() + randid; } else { - user = "dummy" + rand.nextInt(); + user = "dummy" + randid; } return fs.makeQualified(new Path(stagingRootDir, user+"/.staging")).toString(); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobID.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobID.java index 09dc1cb6e1..c5007d9cd8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobID.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobID.java @@ -32,7 +32,7 @@ * the job. JobID consists of two parts. First part * represents the jobtracker identifier, so that jobID to jobtracker map * is defined. For cluster setup this string is the jobtracker - * start time, for local setting, it is "local". + * start time, for local setting, it is "local" and a random number. * Second part of the JobID is the job number.
* An example JobID is : * job_200707121733_0003 , which represents the third job