big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-15988. DynamoDBMetadataStore#innerGet should support empty directory flag when using authoritative listings. Contributed by Gabor Bota.

Browse Source
This commit is contained in:
Sean Mackrory 2018-12-12 09:29:53 -07:00
parent d44b37d7ac
commit 82b798581d
2 changed files with 57 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
View File

@ -576,12 +576,16 @@ private DDBPathMetadata innerGet(Path path, boolean wantEmptyDirectoryFlag)
path.toString(), path.toString(),
true, true,
() -> table.query(spec).iterator().hasNext()); () -> table.query(spec).iterator().hasNext());
// When this class has support for authoritative
// (fully-cached) directory listings, we may also be able to answer // If directory is authoritative, we can set the empty directory flag
// TRUE here. Until then, we don't know if we have full listing or // to TRUE or FALSE. Otherwise FALSE, or UNKNOWN.
// not, thus the UNKNOWN here: if(meta.isAuthoritativeDir()) {
meta.setIsEmptyDirectory( meta.setIsEmptyDirectory(
hasChildren ? Tristate.FALSE : Tristate.UNKNOWN); hasChildren ? Tristate.FALSE : Tristate.TRUE);
} else {
meta.setIsEmptyDirectory(
hasChildren ? Tristate.FALSE : Tristate.UNKNOWN);
}
} }
} }

47
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestDynamoDBMetadataStore.java
View File

@ -687,6 +687,53 @@ public void testTableTagging() throws IOException {
} }
} }
@Test
public void testGetEmptyDirFlagCanSetTrue() throws IOException {
boolean authoritativeDirectoryListing = true;
testGetEmptyDirFlagCanSetTrueOrUnknown(authoritativeDirectoryListing);
}
@Test
public void testGetEmptyDirFlagCanSetUnknown() throws IOException {
boolean authoritativeDirectoryListing = false;
testGetEmptyDirFlagCanSetTrueOrUnknown(authoritativeDirectoryListing);
}
private void testGetEmptyDirFlagCanSetTrueOrUnknown(boolean auth)
throws IOException {
// setup
final DynamoDBMetadataStore ms = getDynamoMetadataStore();
String rootPath = "/testAuthoritativeEmptyDirFlag"+ UUID.randomUUID();
String filePath = rootPath + "/file1";
final Path dirToPut = fileSystem.makeQualified(new Path(rootPath));
final Path fileToPut = fileSystem.makeQualified(new Path(filePath));
// Create non-auth DirListingMetadata
DirListingMetadata dlm =
new DirListingMetadata(dirToPut, new ArrayList<>(), auth);
if(auth){
assertEquals(Tristate.TRUE, dlm.isEmpty());
} else {
assertEquals(Tristate.UNKNOWN, dlm.isEmpty());
}
assertEquals(auth, dlm.isAuthoritative());
// Test with non-authoritative listing, empty dir
ms.put(dlm);
final PathMetadata pmdResultEmpty = ms.get(dirToPut, true);
if(auth){
assertEquals(Tristate.TRUE, pmdResultEmpty.isEmptyDirectory());
} else {
assertEquals(Tristate.UNKNOWN, pmdResultEmpty.isEmptyDirectory());
}
// Test with non-authoritative listing, non-empty dir
dlm.put(basicFileStatus(fileToPut, 1, false));
ms.put(dlm);
final PathMetadata pmdResultNotEmpty = ms.get(dirToPut, true);
assertEquals(Tristate.FALSE, pmdResultNotEmpty.isEmptyDirectory());
}
/** /**
* This validates the table is created and ACTIVE in DynamoDB. * This validates the table is created and ACTIVE in DynamoDB.
* *