big-data/hadoop
big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Projects Wiki Activity

MAPREDUCE-3982. Fixed FileOutputCommitter to not err out for an 'empty-job' whose tasks don't write any outputs. Contributed by Robert Joseph Evans.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1299047 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2012-03-09 21:13:42 +00:00
parent 1c534d0f9d
commit 558cc8c0bf
3 changed files with 29 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -304,6 +304,9 @@ Release 0.23.2 - UNRELEASED
MAPREDUCE-3975. Default value not set for Configuration parameter MAPREDUCE-3975. Default value not set for Configuration parameter
mapreduce.job.local.dir (Eric Payne via bobby) mapreduce.job.local.dir (Eric Payne via bobby)
MAPREDUCE-3982. Fixed FileOutputCommitter to not err out for an 'empty-job'
whose tasks don't write any outputs. (Robert Joseph Evans via vinodkv)
Release 0.23.1 - 2012-02-17 Release 0.23.1 - 2012-02-17
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

8
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java
View File

@ -278,11 +278,11 @@ public class FileOutputCommitter extends OutputCommitter {
*/ */
public void setupJob(JobContext context) throws IOException { public void setupJob(JobContext context) throws IOException {
if (hasOutputPath()) { if (hasOutputPath()) {
Path pendingJobAttemptsPath = getPendingJobAttemptsPath(); Path jobAttemptPath = getJobAttemptPath(context);
FileSystem fs = pendingJobAttemptsPath.getFileSystem( FileSystem fs = jobAttemptPath.getFileSystem(
context.getConfiguration()); context.getConfiguration());
if (!fs.mkdirs(pendingJobAttemptsPath)) { if (!fs.mkdirs(jobAttemptPath)) {
LOG.error("Mkdirs failed to create " + pendingJobAttemptsPath); LOG.error("Mkdirs failed to create " + jobAttemptPath);
} }
} else { } else {
LOG.warn("Output Path is null in setupJob()"); LOG.warn("Output Path is null in setupJob()");

22
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestFileOutputCommitter.java
View File

@ -123,6 +123,28 @@ public class TestFileOutputCommitter extends TestCase {
FileUtil.fullyDelete(new File(outDir.toString())); FileUtil.fullyDelete(new File(outDir.toString()));
} }
public void testEmptyOutput() throws Exception {
Job job = Job.getInstance();
FileOutputFormat.setOutputPath(job, outDir);
Configuration conf = job.getConfiguration();
conf.set(MRJobConfig.TASK_ATTEMPT_ID, attempt);
JobContext jContext = new JobContextImpl(conf, taskID.getJobID());
TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
FileOutputCommitter committer = new FileOutputCommitter(outDir, tContext);
// setup
committer.setupJob(jContext);
committer.setupTask(tContext);
// Do not write any output
// do commit
committer.commitTask(tContext);
committer.commitJob(jContext);
FileUtil.fullyDelete(new File(outDir.toString()));
}
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public void testAbort() throws IOException, InterruptedException { public void testAbort() throws IOException, InterruptedException {
Job job = Job.getInstance(); Job job = Job.getInstance();