HADOOP-18582. Addendum: Skip unnecessary cleanup logic in DistCp. (#5409)
Followup to the original HADOOP-18582. Temporary path cleanup is re-enabled for -append jobs as these will create temporary files when creating or overwriting files. Contributed by Ayush Saxena
This commit is contained in:
parent
fb31393b65
commit
e8a6b2c2c4
@ -156,10 +156,7 @@ private void cleanupTempFiles(JobContext context) {
|
||||
|
||||
final boolean directWrite = conf.getBoolean(
|
||||
DistCpOptionSwitch.DIRECT_WRITE.getConfigLabel(), false);
|
||||
final boolean append = conf.getBoolean(
|
||||
DistCpOptionSwitch.APPEND.getConfigLabel(), false);
|
||||
final boolean useTempTarget = !append && !directWrite;
|
||||
if (!useTempTarget) {
|
||||
if (directWrite) {
|
||||
return;
|
||||
}
|
||||
|
||||
|
@ -586,13 +586,11 @@ fs, new Path(targetBase + srcFilename),
|
||||
|
||||
@Test
|
||||
public void testCommitWithCleanupTempFiles() throws IOException {
|
||||
testCommitWithCleanup(true, false);
|
||||
testCommitWithCleanup(false, true);
|
||||
testCommitWithCleanup(true, true);
|
||||
testCommitWithCleanup(false, false);
|
||||
testCommitWithCleanup(true);
|
||||
testCommitWithCleanup(false);
|
||||
}
|
||||
|
||||
private void testCommitWithCleanup(boolean append, boolean directWrite)throws IOException {
|
||||
private void testCommitWithCleanup(boolean directWrite) throws IOException {
|
||||
TaskAttemptContext taskAttemptContext = getTaskAttemptContext(config);
|
||||
JobID jobID = taskAttemptContext.getTaskAttemptID().getJobID();
|
||||
JobContext jobContext = new JobContextImpl(
|
||||
@ -611,7 +609,7 @@ private void testCommitWithCleanup(boolean append, boolean directWrite)throws IO
|
||||
DistCpOptions options = new DistCpOptions.Builder(
|
||||
Collections.singletonList(new Path(sourceBase)),
|
||||
new Path("/out"))
|
||||
.withAppend(append)
|
||||
.withAppend(true)
|
||||
.withSyncFolder(true)
|
||||
.withDirectWrite(directWrite)
|
||||
.build();
|
||||
@ -631,7 +629,7 @@ private void testCommitWithCleanup(boolean append, boolean directWrite)throws IO
|
||||
null, taskAttemptContext);
|
||||
committer.commitJob(jobContext);
|
||||
|
||||
if (append || directWrite) {
|
||||
if (directWrite) {
|
||||
ContractTestUtils.assertPathExists(fs, "Temp files should not be cleanup with append or direct option",
|
||||
tempFilePath);
|
||||
} else {
|
||||
|
Loading…
Reference in New Issue
Block a user