big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-18582. Addendum: Skip unnecessary cleanup logic in DistCp. (#5409)

Browse Source 
Followup to the original HADOOP-18582.

Temporary path cleanup is re-enabled for -append jobs
as these will create temporary files when creating or overwriting files.

Contributed by Ayush Saxena
This commit is contained in:
Ayush Saxena 2023-02-23 00:59:41 +05:30 committed by GitHub
parent fb31393b65
commit e8a6b2c2c4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 6 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
View File

@ -156,10 +156,7 @@ private void cleanupTempFiles(JobContext context) {
final boolean directWrite = conf.getBoolean( final boolean directWrite = conf.getBoolean(
DistCpOptionSwitch.DIRECT_WRITE.getConfigLabel(), false); DistCpOptionSwitch.DIRECT_WRITE.getConfigLabel(), false);
final boolean append = conf.getBoolean( if (directWrite) {
DistCpOptionSwitch.APPEND.getConfigLabel(), false);
final boolean useTempTarget = !append && !directWrite;
if (!useTempTarget) {
return; return;
} }

12
hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java
View File

@ -586,13 +586,11 @@ fs, new Path(targetBase + srcFilename),
@Test @Test
public void testCommitWithCleanupTempFiles() throws IOException { public void testCommitWithCleanupTempFiles() throws IOException {
testCommitWithCleanup(true, false); testCommitWithCleanup(true);
testCommitWithCleanup(false, true); testCommitWithCleanup(false);
testCommitWithCleanup(true, true);
testCommitWithCleanup(false, false);
} }
private void testCommitWithCleanup(boolean append, boolean directWrite)throws IOException { private void testCommitWithCleanup(boolean directWrite) throws IOException {
TaskAttemptContext taskAttemptContext = getTaskAttemptContext(config); TaskAttemptContext taskAttemptContext = getTaskAttemptContext(config);
JobID jobID = taskAttemptContext.getTaskAttemptID().getJobID(); JobID jobID = taskAttemptContext.getTaskAttemptID().getJobID();
JobContext jobContext = new JobContextImpl( JobContext jobContext = new JobContextImpl(
@ -611,7 +609,7 @@ private void testCommitWithCleanup(boolean append, boolean directWrite)throws IO
DistCpOptions options = new DistCpOptions.Builder( DistCpOptions options = new DistCpOptions.Builder(
Collections.singletonList(new Path(sourceBase)), Collections.singletonList(new Path(sourceBase)),
new Path("/out")) new Path("/out"))
.withAppend(append) .withAppend(true)
.withSyncFolder(true) .withSyncFolder(true)
.withDirectWrite(directWrite) .withDirectWrite(directWrite)
.build(); .build();
@ -631,7 +629,7 @@ private void testCommitWithCleanup(boolean append, boolean directWrite)throws IO
null, taskAttemptContext); null, taskAttemptContext);
committer.commitJob(jobContext); committer.commitJob(jobContext);
if (append || directWrite) { if (directWrite) {
ContractTestUtils.assertPathExists(fs, "Temp files should not be cleanup with append or direct option", ContractTestUtils.assertPathExists(fs, "Temp files should not be cleanup with append or direct option",
tempFilePath); tempFilePath);
} else { } else {