big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-14428. s3a: mkdir appears to be broken. Contributed by Mingliang Liu

Browse Source
This commit is contained in:
Mingliang Liu 2017-05-24 14:44:27 -07:00
parent abdd609e51
commit 6aeda55bb8
2 changed files with 23 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMkdirTest.java
View File

@ -113,18 +113,25 @@ public void testMkdirSlashHandling() throws Throwable {
describe("verify mkdir slash handling"); describe("verify mkdir slash handling");
FileSystem fs = getFileSystem(); FileSystem fs = getFileSystem();
// No trailing slash final Path[] paths = new Path[] {
assertTrue(fs.mkdirs(path("testmkdir/a"))); path("testMkdirSlashHandling/a"), // w/o trailing slash
assertPathExists("mkdir without trailing slash failed", path("testMkdirSlashHandling/b/"), // w/ trailing slash
path("testmkdir/a")); // unqualified w/o trailing slash
new Path(getContract().getTestPath() + "/testMkdirSlashHandling/c"),
// With trailing slash // unqualified w/ trailing slash
assertTrue(fs.mkdirs(path("testmkdir/b/"))); new Path(getContract().getTestPath() + "/testMkdirSlashHandling/d/"),
assertPathExists("mkdir with trailing slash failed", path("testmkdir/b/")); // unqualified w/ multiple trailing slashes
new Path(getContract().getTestPath() + "/testMkdirSlashHandling/e///")
// Mismatched slashes };
assertPathExists("check path existence without trailing slash failed", for (Path path : paths) {
path("testmkdir/b")); assertTrue(fs.mkdirs(path));
assertPathExists(path + " does not exist after mkdirs", path);
assertIsDirectory(path);
if (path.toString().endsWith("/")) {
String s = path.toString().substring(0, path.toString().length() - 1);
assertIsDirectory(new Path(s));
}
}
} }
@Test @Test

5
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
View File

@ -1587,7 +1587,9 @@ private boolean innerMkdirs(Path f, FsPermission permission)
String key = pathToKey(f); String key = pathToKey(f);
createFakeDirectory(key); createFakeDirectory(key);
deleteUnnecessaryFakeDirectories(f.getParent()); // this is complicated because getParent(a/b/c/) returns a/b/c, but
// we want a/b. See HADOOP-14428 for more details.
deleteUnnecessaryFakeDirectories(new Path(f.toString()).getParent());
return true; return true;
} }
} }
@ -1971,6 +1973,7 @@ private void deleteUnnecessaryFakeDirectories(Path path) {
while (!path.isRoot()) { while (!path.isRoot()) {
String key = pathToKey(path); String key = pathToKey(path);
key = (key.endsWith("/")) ? key : (key + "/"); key = (key.endsWith("/")) ? key : (key + "/");
LOG.trace("To delete unnecessary fake directory {} for {}", key, path);
keysToRemove.add(new DeleteObjectsRequest.KeyVersion(key)); keysToRemove.add(new DeleteObjectsRequest.KeyVersion(key));
path = path.getParent(); path = path.getParent();
} }