big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-17328. LazyPersist Overwrite fails in direct write mode. (#2413)

Browse Source
This commit is contained in:
Ayush Saxena 2020-10-26 22:07:37 +05:30 committed by GitHub
parent 12c908c827
commit 872440610f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 27 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
View File

@ -54,6 +54,7 @@
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY;
import static org.apache.hadoop.fs.CreateFlag.CREATE; import static org.apache.hadoop.fs.CreateFlag.CREATE;
import static org.apache.hadoop.fs.CreateFlag.LAZY_PERSIST; import static org.apache.hadoop.fs.CreateFlag.LAZY_PERSIST;
import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
/** /**
* Provides: argument processing to ensure the destination is valid * Provides: argument processing to ensure the destination is valid
@ -515,7 +516,8 @@ FSDataOutputStream create(PathData item, boolean lazyPersist)
defaultBlockSize = getDefaultBlockSize(item.path); defaultBlockSize = getDefaultBlockSize(item.path);
} }
EnumSet<CreateFlag> createFlags = EnumSet.of(CREATE, LAZY_PERSIST); EnumSet<CreateFlag> createFlags =
EnumSet.of(CREATE, LAZY_PERSIST, OVERWRITE);
return create(item.path, return create(item.path,
FsPermission.getFileDefault().applyUMask( FsPermission.getFileDefault().applyUMask(
FsPermission.getUMask(getConf())), FsPermission.getUMask(getConf())),

24
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellCopy.java
View File

@ -23,6 +23,7 @@
import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.CoreMatchers.not;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertThat; import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
@ -697,4 +698,27 @@ public void testPutSrcFileNoPerm()
lfs.setPermission(src, new FsPermission((short)0755)); lfs.setPermission(src, new FsPermission((short)0755));
} }
} }
@Test
public void testLazyPersistDirectOverwrite() throws Exception {
Path testRoot = new Path(testRootDir, "testLazyPersistDirectOverwrite");
try {
lfs.delete(testRoot, true);
lfs.mkdirs(testRoot);
Path filePath = new Path(testRoot, new Path("srcFile"));
lfs.create(filePath).close();
// Put with overwrite in direct mode.
String[] argv =
new String[] {"-put", "-f", "-l", "-d", filePath.toString(),
filePath.toString()};
assertEquals(0, shell.run(argv));
// Put without overwrite in direct mode shouldn't be success.
argv = new String[] {"-put", "-l", "-d", filePath.toString(),
filePath.toString()};
assertNotEquals(0, shell.run(argv));
} finally {
lfs.delete(testRoot, true);
}
}
} }