big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-14255. S3A to delete unnecessary fake directory objects in mkdirs(). Contributed by Mingliang Liu

Browse Source
This commit is contained in:
Mingliang Liu 2017-03-30 13:03:34 -07:00
parent c3375175d6
commit b053fdc547
2 changed files with 43 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMkdirTest.java
View File

@ -126,4 +126,46 @@ public void testMkdirSlashHandling() throws Throwable {
assertPathExists("check path existence without trailing slash failed",
path("testmkdir/b"));
}
@Test
public void testMkdirsPopulatingAllNonexistentAncestors() throws IOException {
describe("Verify mkdir will populate all its non-existent ancestors");
final FileSystem fs = getFileSystem();
final Path parent = path("testMkdirsPopulatingAllNonexistentAncestors");
assertTrue(fs.mkdirs(parent));
assertPathExists(parent + " should exist before making nested dir", parent);
Path nested = path(parent + "/a/b/c/d/e/f/g/h/i/j/k/L");
assertTrue(fs.mkdirs(nested));
while (nested != null && !nested.equals(parent) && !nested.isRoot()) {
assertPathExists(nested + " nested dir should exist", nested);
nested = nested.getParent();
}
}
@Test
public void testMkdirsDoesNotRemoveParentDirectories() throws IOException {
describe("Verify mkdir will make its parent existent");
final FileSystem fs = getFileSystem();
final Path parent = path("testMkdirsDoesNotRemoveParentDirectories");
assertTrue(fs.mkdirs(parent));
Path p = parent;
for (int i = 0; i < 10; i++) {
assertTrue(fs.mkdirs(p));
assertPathExists(p + " should exist after mkdir(" + p + ")", p);
p = path(p + "/dir-" + i);
}
// After mkdirs(sub-directory), its parent directory still exists
p = p.getParent();
while (p != null && !p.equals(parent) && !p.isRoot()) {
assertPathExists("Path " + p + " should exist", p);
assertIsDirectory(p);
p = p.getParent();
}
}
}

3
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
View File

@ -1525,8 +1525,6 @@ public boolean mkdirs(Path path, FsPermission permission) throws IOException,
* @throws IOException other IO problems
* @throws AmazonClientException on failures inside the AWS SDK
*/
// TODO: If we have created an empty file at /foo/bar and we then call
// mkdirs for /foo/bar/baz/roo what happens to the empty file /foo/bar/?
private boolean innerMkdirs(Path f, FsPermission permission)
throws IOException, FileAlreadyExistsException, AmazonClientException {
LOG.debug("Making directory: {}", f);
@ -1561,6 +1559,7 @@ private boolean innerMkdirs(Path f, FsPermission permission)
String key = pathToKey(f);
createFakeDirectory(key);
deleteUnnecessaryFakeDirectories(f.getParent());
return true;
}
}