From 6571d39a74ec183046b8f8d6b3181dcd9b7f2e7f Mon Sep 17 00:00:00 2001 From: Thomas White Date: Mon, 12 Dec 2011 18:29:20 +0000 Subject: [PATCH] MAPREDUCE-3527. Fix minor API incompatibilities between 1.0 and 0.23. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1213350 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../hadoop/mapred/InvalidJobConfException.java | 8 ++++++++ .../java/org/apache/hadoop/mapred/JobClient.java | 10 ++++++++++ .../org/apache/hadoop/mapred/RunningJob.java | 7 +++++++ .../mapred/lib/CombineFileInputFormat.java | 16 ++++++++++++++++ .../apache/hadoop/mapreduce/TaskAttemptID.java | 6 ++++++ .../java/org/apache/hadoop/mapreduce/TaskID.java | 6 ++++++ .../lib/input/KeyValueLineRecordReader.java | 2 ++ 8 files changed, 58 insertions(+) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index e6477d83ca..0285fe80ef 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -261,6 +261,9 @@ Release 0.23.1 - Unreleased MAPREDUCE-3519. Fixed a deadlock in NodeManager LocalDirectories's handling service. (Ravi Gummadi via vinodkv) + MAPREDUCE-3527. Fix minor API incompatibilities between 1.0 and 0.23. + (tomwhite) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/InvalidJobConfException.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/InvalidJobConfException.java index 3f1a74e63c..ebd1e0f6b9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/InvalidJobConfException.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/InvalidJobConfException.java @@ -41,5 +41,13 @@ public InvalidJobConfException() { public InvalidJobConfException(String msg) { super(msg); } + + public InvalidJobConfException(String msg, Throwable t) { + super(msg, t); + } + + public InvalidJobConfException(Throwable t) { + super(t); + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java index 1b0b453ddd..cc586bf2f9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java @@ -420,6 +420,16 @@ public boolean isRetired() throws IOException { boolean monitorAndPrintJob() throws IOException, InterruptedException { return job.monitorAndPrintJob(); } + + @Override + public String getFailureInfo() throws IOException { + try { + return job.getStatus().getFailureInfo(); + } catch (InterruptedException ie) { + throw new IOException(ie); + } + } + } Cluster cluster; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/RunningJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/RunningJob.java index b3af4f6c98..53b7188e9b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/RunningJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/RunningJob.java @@ -223,4 +223,11 @@ public TaskCompletionEvent[] getTaskCompletionEvents(int startFrom) * @throws IOException */ public boolean isRetired() throws IOException; + + /** + * Get failure info for the job. + * @return the failure info for the job. + * @throws IOException + */ + public String getFailureInfo() throws IOException; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java index 96a30b010c..35dd3d7e5c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java @@ -23,7 +23,10 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.mapred.FileInputFormat; import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; @@ -115,5 +118,18 @@ public org.apache.hadoop.mapreduce.RecordReader createRecordReader( TaskAttemptContext context) throws IOException { return null; } + + /** List input directories. + * Subclasses may override to, e.g., select only files matching a regular + * expression. + * + * @param job the job to list input paths for + * @return array of FileStatus objects + * @throws IOException if zero items. + */ + protected FileStatus[] listStatus(JobConf job) throws IOException { + List result = super.listStatus(new Job(job)); + return result.toArray(new FileStatus[result.size()]); + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskAttemptID.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskAttemptID.java index 16fd96210a..70fcf1024c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskAttemptID.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskAttemptID.java @@ -90,6 +90,12 @@ public JobID getJobID() { public TaskID getTaskID() { return taskId; } + + /**Returns whether this TaskID is a map ID */ + @Deprecated + public boolean isMap() { + return taskId.isMap(); + } /**Returns the TaskType of the TaskAttemptID */ public TaskType getTaskType() { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskID.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskID.java index a09282f2bd..c98ec904fa 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskID.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskID.java @@ -100,6 +100,12 @@ public TaskID() { public JobID getJobID() { return jobId; } + + /**Returns whether this TaskID is a map ID */ + @Deprecated + public boolean isMap() { + return type == TaskType.MAP; + } /** * Get the type of the task diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/KeyValueLineRecordReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/KeyValueLineRecordReader.java index d44bccd911..6d661211cf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/KeyValueLineRecordReader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/KeyValueLineRecordReader.java @@ -50,6 +50,8 @@ public class KeyValueLineRecordReader extends RecordReader { private Text value; + public Class getKeyClass() { return Text.class; } + public KeyValueLineRecordReader(Configuration conf) throws IOException {