diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java index 03dfed2994..3eda24fbe2 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java @@ -93,11 +93,8 @@ protected void preCommitJob(JobContext context, Configuration fsConf = fs.getConf(); switch (getConflictResolutionMode(context, fsConf)) { case FAIL: - // this was checked in setupJob, but this avoids some cases where - // output was created while the job was processing - if (fs.exists(outputPath)) { - throw new PathExistsException(outputPath.toString(), E_DEST_EXISTS); - } + // this was checked in setupJob; temporary files may have been + // created, so do not check again. break; case APPEND: // do nothing diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java index 4f0189e891..f5b8d1f872 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java @@ -67,15 +67,14 @@ protected void verifyFailureConflictOutcome() throws Exception { pathExists(mockS3, OUTPUT_PATH); final DirectoryStagingCommitter committer = newJobCommitter(); + // this should fail intercept(PathExistsException.class, InternalCommitterConstants.E_DEST_EXISTS, "Should throw an exception because the path exists", () -> committer.setupJob(getJob())); - intercept(PathExistsException.class, - InternalCommitterConstants.E_DEST_EXISTS, - "Should throw an exception because the path exists", - () -> committer.commitJob(getJob())); + // but there are no checks in job commit (HADOOP-15469) + committer.commitJob(getJob()); reset(mockS3); pathDoesNotExist(mockS3, OUTPUT_PATH); @@ -87,7 +86,6 @@ protected void verifyFailureConflictOutcome() throws Exception { reset(mockS3); pathDoesNotExist(mockS3, OUTPUT_PATH); committer.commitJob(getJob()); - verifyExistenceChecked(mockS3, OUTPUT_PATH); verifyCompletion(mockS3); }