diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index f8eba04bbc..f3167f283a 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1368,6 +1368,16 @@
+
+ fs.s3a.metadatastore.authoritative.dir.ttl
+ 3600000
+
+ This value sets how long a directory listing in the MS is considered as
+ authoritative. The value is in milliseconds.
+ MetadataStore should be authoritative to use this configuration knob.
+
+
+
fs.s3a.metadatastore.impl
org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index 3fc25daaec..9a71f32cd1 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -21,6 +21,8 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import java.util.concurrent.TimeUnit;
+
/**
* All the constants used with the {@link S3AFileSystem}.
*
@@ -327,6 +329,14 @@ private Constants() {
"fs.s3a.metadatastore.authoritative";
public static final boolean DEFAULT_METADATASTORE_AUTHORITATIVE = false;
+ /**
+ * How long a directory listing in the MS is considered as authoritative.
+ */
+ public static final String METADATASTORE_AUTHORITATIVE_DIR_TTL =
+ "fs.s3a.metadatastore.authoritative.dir.ttl";
+ public static final long DEFAULT_METADATASTORE_AUTHORITATIVE_DIR_TTL =
+ TimeUnit.MINUTES.toMillis(60);
+
/** read ahead buffer size to prevent connection re-establishments. */
public static final String READAHEAD_RANGE = "fs.s3a.readahead.range";
public static final long DEFAULT_READAHEAD_RANGE = 64 * 1024;
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index e817f0d55d..df0ec5d462 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -205,6 +205,8 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities {
private AWSCredentialProviderList credentials;
+ private S3Guard.ITtlTimeProvider ttlTimeProvider;
+
/** Add any deprecated keys. */
@SuppressWarnings("deprecation")
private static void addDeprecatedKeys() {
@@ -345,6 +347,9 @@ public void initialize(URI name, Configuration originalConf)
getMetadataStore(), allowAuthoritative);
}
initMultipartUploads(conf);
+ long authDirTtl = conf.getLong(METADATASTORE_AUTHORITATIVE_DIR_TTL,
+ DEFAULT_METADATASTORE_AUTHORITATIVE_DIR_TTL);
+ ttlTimeProvider = new S3Guard.TtlTimeProvider(authDirTtl);
} catch (AmazonClientException e) {
throw translateException("initializing ", new Path(name), e);
}
@@ -1907,7 +1912,8 @@ public FileStatus[] innerListStatus(Path f) throws FileNotFoundException,
key = key + '/';
}
- DirListingMetadata dirMeta = metadataStore.listChildren(path);
+ DirListingMetadata dirMeta =
+ S3Guard.listChildrenWithTtl(metadataStore, path, ttlTimeProvider);
if (allowAuthoritative && dirMeta != null && dirMeta.isAuthoritative()) {
return S3Guard.dirMetaToStatuses(dirMeta);
}
@@ -1925,7 +1931,7 @@ public FileStatus[] innerListStatus(Path f) throws FileNotFoundException,
result.add(files.next());
}
return S3Guard.dirListingUnion(metadataStore, path, result, dirMeta,
- allowAuthoritative);
+ allowAuthoritative, ttlTimeProvider);
} else {
LOG.debug("Adding: rd (not a dir): {}", path);
FileStatus[] stats = new FileStatus[1];
@@ -2135,7 +2141,8 @@ S3AFileStatus innerGetFileStatus(final Path f,
// We have a definitive true / false from MetadataStore, we are done.
return S3AFileStatus.fromFileStatus(msStatus, pm.isEmptyDirectory());
} else {
- DirListingMetadata children = metadataStore.listChildren(path);
+ DirListingMetadata children =
+ S3Guard.listChildrenWithTtl(metadataStore, path, ttlTimeProvider);
if (children != null) {
tombstones = children.listTombstones();
}
@@ -3122,7 +3129,8 @@ private RemoteIterator innerListFiles(Path f, boolean
tombstones = metadataStoreListFilesIterator.listTombstones();
cachedFilesIterator = metadataStoreListFilesIterator;
} else {
- DirListingMetadata meta = metadataStore.listChildren(path);
+ DirListingMetadata meta =
+ S3Guard.listChildrenWithTtl(metadataStore, path, ttlTimeProvider);
if (meta != null) {
tombstones = meta.listTombstones();
} else {
@@ -3195,7 +3203,9 @@ public RemoteIterator listLocatedStatus(final Path f,
final String key = maybeAddTrailingSlash(pathToKey(path));
final Listing.FileStatusAcceptor acceptor =
new Listing.AcceptAllButSelfAndS3nDirs(path);
- DirListingMetadata meta = metadataStore.listChildren(path);
+ DirListingMetadata meta =
+ S3Guard.listChildrenWithTtl(metadataStore, path,
+ ttlTimeProvider);
final RemoteIterator cachedFileStatusIterator =
listing.createProvidedFileStatusIterator(
S3Guard.dirMetaToStatuses(meta), filter, acceptor);
@@ -3346,4 +3356,14 @@ public AWSCredentialProviderList shareCredentials(final String purpose) {
LOG.debug("Sharing credentials for: {}", purpose);
return credentials.share();
}
+
+ @VisibleForTesting
+ protected S3Guard.ITtlTimeProvider getTtlTimeProvider() {
+ return ttlTimeProvider;
+ }
+
+ @VisibleForTesting
+ protected void setTtlTimeProvider(S3Guard.ITtlTimeProvider ttlTimeProvider) {
+ this.ttlTimeProvider = ttlTimeProvider;
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DDBPathMetadata.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DDBPathMetadata.java
index a67fc4e22f..78568dc4bb 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DDBPathMetadata.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DDBPathMetadata.java
@@ -30,14 +30,10 @@ public class DDBPathMetadata extends PathMetadata {
private boolean isAuthoritativeDir;
- public DDBPathMetadata(PathMetadata pmd, boolean isAuthoritativeDir) {
- super(pmd.getFileStatus(), pmd.isEmptyDirectory(), pmd.isDeleted());
- this.isAuthoritativeDir = isAuthoritativeDir;
- }
-
public DDBPathMetadata(PathMetadata pmd) {
super(pmd.getFileStatus(), pmd.isEmptyDirectory(), pmd.isDeleted());
this.isAuthoritativeDir = false;
+ this.setLastUpdated(pmd.getLastUpdated());
}
public DDBPathMetadata(FileStatus fileStatus) {
@@ -52,9 +48,10 @@ public DDBPathMetadata(FileStatus fileStatus, Tristate isEmptyDir,
}
public DDBPathMetadata(FileStatus fileStatus, Tristate isEmptyDir,
- boolean isDeleted, boolean isAuthoritativeDir) {
+ boolean isDeleted, boolean isAuthoritativeDir, long lastUpdated) {
super(fileStatus, isEmptyDir, isDeleted);
this.isAuthoritativeDir = isAuthoritativeDir;
+ this.setLastUpdated(lastUpdated);
}
public boolean isAuthoritativeDir() {
@@ -74,4 +71,11 @@ public boolean equals(Object o) {
return super.hashCode();
}
+ @Override public String toString() {
+ return "DDBPathMetadata{" +
+ "isAuthoritativeDir=" + isAuthoritativeDir +
+ ", lastUpdated=" + this.getLastUpdated() +
+ ", PathMetadata=" + super.toString() +
+ '}';
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java
index e5b4fb541a..e7f0207268 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java
@@ -42,7 +42,7 @@
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
-public class DirListingMetadata {
+public class DirListingMetadata extends ExpirableMetadata {
/**
* Convenience parameter for passing into constructor.
@@ -69,7 +69,7 @@ public class DirListingMetadata {
* the full and authoritative listing of all files in the directory.
*/
public DirListingMetadata(Path path, Collection listing,
- boolean isAuthoritative) {
+ boolean isAuthoritative, long lastUpdated) {
checkPathAbsolute(path);
this.path = path;
@@ -82,6 +82,12 @@ public DirListingMetadata(Path path, Collection listing,
}
}
this.isAuthoritative = isAuthoritative;
+ this.setLastUpdated(lastUpdated);
+ }
+
+ public DirListingMetadata(Path path, Collection listing,
+ boolean isAuthoritative) {
+ this(path, listing, isAuthoritative, 0);
}
/**
@@ -91,6 +97,7 @@ public DirListingMetadata(Path path, Collection listing,
public DirListingMetadata(DirListingMetadata d) {
path = d.path;
isAuthoritative = d.isAuthoritative;
+ this.setLastUpdated(d.getLastUpdated());
listMap = new ConcurrentHashMap<>(d.listMap);
}
@@ -125,7 +132,8 @@ public DirListingMetadata withoutTombstones() {
filteredList.add(meta);
}
}
- return new DirListingMetadata(path, filteredList, isAuthoritative);
+ return new DirListingMetadata(path, filteredList, isAuthoritative,
+ this.getLastUpdated());
}
/**
@@ -231,6 +239,7 @@ public String toString() {
"path=" + path +
", listMap=" + listMap +
", isAuthoritative=" + isAuthoritative +
+ ", lastUpdated=" + this.getLastUpdated() +
'}';
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
index 7c826c11db..5716cfa062 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
@@ -632,7 +632,8 @@ public DirListingMetadata listChildren(final Path path) throws IOException {
return (metas.isEmpty() && dirPathMeta == null)
? null
- : new DirListingMetadata(path, metas, isAuthoritative);
+ : new DirListingMetadata(path, metas, isAuthoritative,
+ dirPathMeta.getLastUpdated());
});
}
@@ -864,7 +865,7 @@ Collection fullPathsToPut(DDBPathMetadata meta)
if (!itemExists(item)) {
final FileStatus status = makeDirStatus(path, username);
metasToPut.add(new DDBPathMetadata(status, Tristate.FALSE, false,
- meta.isAuthoritativeDir()));
+ meta.isAuthoritativeDir(), meta.getLastUpdated()));
path = path.getParent();
} else {
break;
@@ -907,7 +908,7 @@ public void put(DirListingMetadata meta) throws IOException {
Path path = meta.getPath();
DDBPathMetadata ddbPathMeta =
new DDBPathMetadata(makeDirStatus(path, username), meta.isEmpty(),
- false, meta.isAuthoritative());
+ false, meta.isAuthoritative(), meta.getLastUpdated());
// First add any missing ancestors...
final Collection metasToPut = fullPathsToPut(ddbPathMeta);
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PathMetadata.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PathMetadata.java
index 2a0219e4ce..56645fead7 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PathMetadata.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PathMetadata.java
@@ -31,7 +31,7 @@
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
-public class PathMetadata {
+public class PathMetadata extends ExpirableMetadata {
private final FileStatus fileStatus;
private Tristate isEmptyDirectory;
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PathMetadataDynamoDBTranslation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PathMetadataDynamoDBTranslation.java
index 46f406fd3e..c6f70bf277 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PathMetadataDynamoDBTranslation.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PathMetadataDynamoDBTranslation.java
@@ -22,7 +22,9 @@
import java.net.URI;
import java.util.Arrays;
import java.util.Collection;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import java.util.stream.Collectors;
import com.amazonaws.services.dynamodbv2.document.Item;
@@ -67,6 +69,11 @@ final class PathMetadataDynamoDBTranslation {
static final String BLOCK_SIZE = "block_size";
static final String IS_DELETED = "is_deleted";
static final String IS_AUTHORITATIVE = "is_authoritative";
+ static final String LAST_UPDATED = "last_updated";
+
+ /** Used while testing backward compatibility. */
+ @VisibleForTesting
+ static final Set IGNORED_FIELDS = new HashSet<>();
/** Table version field {@value} in version marker item. */
@VisibleForTesting
@@ -107,23 +114,7 @@ static Collection attributeDefinitions() {
* @param item DynamoDB item to convert
* @return {@code item} converted to a {@link DDBPathMetadata}
*/
- static DDBPathMetadata itemToPathMetadata(Item item, String username)
- throws IOException {
- return itemToPathMetadata(item, username, false);
- }
-
- /**
- * Converts a DynamoDB item to a {@link DDBPathMetadata}.
- * Can ignore {@code IS_AUTHORITATIVE} flag if {@code ignoreIsAuthFlag} is
- * true.
- *
- * @param item DynamoDB item to convert
- * @param ignoreIsAuthFlag if true, ignore the authoritative flag on item
- * @return {@code item} converted to a {@link DDBPathMetadata}
- */
- static DDBPathMetadata itemToPathMetadata(Item item, String username,
- boolean ignoreIsAuthFlag)
- throws IOException {
+ static DDBPathMetadata itemToPathMetadata(Item item, String username) {
if (item == null) {
return null;
}
@@ -145,11 +136,11 @@ static DDBPathMetadata itemToPathMetadata(Item item, String username,
boolean isDir = item.hasAttribute(IS_DIR) && item.getBoolean(IS_DIR);
boolean isAuthoritativeDir = false;
final FileStatus fileStatus;
+ long lastUpdated = 0;
if (isDir) {
- if (!ignoreIsAuthFlag) {
- isAuthoritativeDir = item.hasAttribute(IS_AUTHORITATIVE)
- && item.getBoolean(IS_AUTHORITATIVE);
- }
+ isAuthoritativeDir = !IGNORED_FIELDS.contains(IS_AUTHORITATIVE)
+ && item.hasAttribute(IS_AUTHORITATIVE)
+ && item.getBoolean(IS_AUTHORITATIVE);
fileStatus = DynamoDBMetadataStore.makeDirStatus(path, username);
} else {
long len = item.hasAttribute(FILE_LENGTH) ? item.getLong(FILE_LENGTH) : 0;
@@ -158,21 +149,16 @@ static DDBPathMetadata itemToPathMetadata(Item item, String username,
fileStatus = new FileStatus(len, false, 1, block, modTime, 0, null,
username, username, path);
}
+ lastUpdated =
+ !IGNORED_FIELDS.contains(LAST_UPDATED)
+ && item.hasAttribute(LAST_UPDATED)
+ ? item.getLong(LAST_UPDATED) : 0;
+
boolean isDeleted =
item.hasAttribute(IS_DELETED) && item.getBoolean(IS_DELETED);
return new DDBPathMetadata(fileStatus, Tristate.UNKNOWN, isDeleted,
- isAuthoritativeDir);
- }
-
- /**
- * Converts a {@link DDBPathMetadata} to a DynamoDB item.
- *
- * @param meta {@link DDBPathMetadata} to convert
- * @return {@code meta} converted to DynamoDB item
- */
- static Item pathMetadataToItem(DDBPathMetadata meta) {
- return pathMetadataToItem(meta, false);
+ isAuthoritativeDir, lastUpdated);
}
/**
@@ -182,17 +168,15 @@ static Item pathMetadataToItem(DDBPathMetadata meta) {
* true.
*
* @param meta {@link DDBPathMetadata} to convert
- * @param ignoreIsAuthFlag if true, ignore the authoritative flag on item
* @return {@code meta} converted to DynamoDB item
*/
- static Item pathMetadataToItem(DDBPathMetadata meta,
- boolean ignoreIsAuthFlag) {
+ static Item pathMetadataToItem(DDBPathMetadata meta) {
Preconditions.checkNotNull(meta);
final FileStatus status = meta.getFileStatus();
final Item item = new Item().withPrimaryKey(pathToKey(status.getPath()));
if (status.isDirectory()) {
item.withBoolean(IS_DIR, true);
- if (!ignoreIsAuthFlag) {
+ if (!IGNORED_FIELDS.contains(IS_AUTHORITATIVE)) {
item.withBoolean(IS_AUTHORITATIVE, meta.isAuthoritativeDir());
}
} else {
@@ -201,6 +185,11 @@ static Item pathMetadataToItem(DDBPathMetadata meta,
.withLong(BLOCK_SIZE, status.getBlockSize());
}
item.withBoolean(IS_DELETED, meta.isDeleted());
+
+ if(!IGNORED_FIELDS.contains(LAST_UPDATED)) {
+ item.withLong(LAST_UPDATED, meta.getLastUpdated());
+ }
+
return item;
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java
index cc55951869..bb8d9b9c57 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java
@@ -194,7 +194,8 @@ public static FileStatus[] dirMetaToStatuses(DirListingMetadata dirMeta) {
*/
public static FileStatus[] dirListingUnion(MetadataStore ms, Path path,
List backingStatuses, DirListingMetadata dirMeta,
- boolean isAuthoritative) throws IOException {
+ boolean isAuthoritative, ITtlTimeProvider timeProvider)
+ throws IOException {
// Fast-path for NullMetadataStore
if (isNullMetadataStore(ms)) {
@@ -241,7 +242,7 @@ public static FileStatus[] dirListingUnion(MetadataStore ms, Path path,
if (changed && isAuthoritative) {
dirMeta.setAuthoritative(true); // This is the full directory contents
- ms.put(dirMeta);
+ S3Guard.putWithTtl(ms, dirMeta, timeProvider);
}
return dirMetaToStatuses(dirMeta);
@@ -282,7 +283,7 @@ public static boolean isNullMetadataStore(MetadataStore ms) {
@Deprecated
@Retries.OnceExceptionsSwallowed
public static void makeDirsOrdered(MetadataStore ms, List dirs,
- String owner, boolean authoritative) {
+ String owner, boolean authoritative, ITtlTimeProvider timeProvider) {
if (dirs == null) {
return;
}
@@ -326,7 +327,7 @@ public static void makeDirsOrdered(MetadataStore ms, List dirs,
children.add(new PathMetadata(prevStatus));
}
dirMeta = new DirListingMetadata(f, children, authoritative);
- ms.put(dirMeta);
+ S3Guard.putWithTtl(ms, dirMeta, timeProvider);
}
pathMetas.add(new PathMetadata(status));
@@ -487,4 +488,56 @@ public static void assertQualified(Path...paths) {
assertQualified(path);
}
}
+
+ /**
+ * This interface is defined for testing purposes.
+ * TTL can be tested by implementing this interface and setting is as
+ * {@code S3Guard.ttlTimeProvider}. By doing this, getNow() can return any
+ * value preferred and flaky tests could be avoided.
+ */
+ public interface ITtlTimeProvider {
+ long getNow();
+ long getAuthoritativeDirTtl();
+ }
+
+ /**
+ * Runtime implementation for TTL Time Provider interface.
+ */
+ public static class TtlTimeProvider implements ITtlTimeProvider {
+ private long authoritativeDirTtl;
+
+ public TtlTimeProvider(long authoritativeDirTtl) {
+ this.authoritativeDirTtl = authoritativeDirTtl;
+ }
+
+ @Override
+ public long getNow() {
+ return System.currentTimeMillis();
+ }
+
+ @Override public long getAuthoritativeDirTtl() {
+ return authoritativeDirTtl;
+ }
+ }
+
+ public static void putWithTtl(MetadataStore ms, DirListingMetadata dirMeta,
+ ITtlTimeProvider timeProvider)
+ throws IOException {
+ dirMeta.setLastUpdated(timeProvider.getNow());
+ ms.put(dirMeta);
+ }
+
+ public static DirListingMetadata listChildrenWithTtl(MetadataStore ms,
+ Path path, ITtlTimeProvider timeProvider)
+ throws IOException {
+ long ttl = timeProvider.getAuthoritativeDirTtl();
+
+ DirListingMetadata dlm = ms.listChildren(path);
+
+ if(dlm != null && dlm.isAuthoritative()
+ && dlm.isExpired(ttl, timeProvider.getNow())) {
+ dlm.setAuthoritative(false);
+ }
+ return dlm;
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md
index a8c8d6cd2c..b4cbd29889 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md
@@ -165,6 +165,17 @@ In particular: **If the Metadata Store is declared as authoritative,
all interactions with the S3 bucket(s) must be through S3A clients sharing
the same Metadata Store**
+It can be configured how long a directory listing in the MetadataStore is
+considered as authoritative. If `((lastUpdated + ttl) <= now)` is false, the
+directory listing is no longer considered authoritative, so the flag will be
+removed on `S3AFileSystem` level.
+
+```xml
+
+ fs.s3a.metadatastore.authoritative.dir.ttl
+ 3600000
+
+```
### 3. Configure the Metadata Store.
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
index 869997b44e..8d9d7b151f 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
@@ -31,6 +31,8 @@
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.s3a.commit.CommitConstants;
+import org.apache.hadoop.fs.s3a.s3guard.MetadataStore;
+import org.apache.hadoop.fs.s3a.s3guard.MetadataStoreCapabilities;
import org.hamcrest.core.Is;
import org.junit.Assert;
import org.junit.Assume;
@@ -46,6 +48,7 @@
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.Callable;
import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
@@ -906,4 +909,14 @@ public static boolean authenticationContains(Configuration conf,
.contains(providerClassname);
}
+ public static boolean metadataStorePersistsAuthoritativeBit(MetadataStore ms)
+ throws IOException {
+ Map diags = ms.getDiagnostics();
+ String persists =
+ diags.get(MetadataStoreCapabilities.PERSISTS_AUTHORITATIVE_BIT);
+ if(persists == null){
+ return false;
+ }
+ return Boolean.valueOf(persists);
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
index 45d6051ddb..27537c03cd 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
@@ -24,7 +24,6 @@
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
-import java.util.Map;
import com.google.common.collect.Sets;
import org.junit.After;
@@ -44,6 +43,9 @@
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.test.HadoopTestBase;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.isMetadataStoreAuthoritative;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.metadataStorePersistsAuthoritativeBit;
+
/**
* Main test class for MetadataStore implementations.
* Implementations should each create a test by subclassing this and
@@ -511,21 +513,13 @@ public void testListChildren() throws Exception {
}
}
- private boolean isMetadataStoreAuthoritative() throws IOException {
- Map diags = ms.getDiagnostics();
- String isAuth =
- diags.get(MetadataStoreCapabilities.PERSISTS_AUTHORITATIVE_BIT);
- if(isAuth == null){
- return false;
- }
- return Boolean.valueOf(isAuth);
- }
+
@Test
public void testListChildrenAuthoritative() throws IOException {
Assume.assumeTrue("MetadataStore should be capable for authoritative "
+ "storage of directories to run this test.",
- isMetadataStoreAuthoritative());
+ metadataStorePersistsAuthoritativeBit(ms));
setupListStatus();
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestPathMetadataDynamoDBTranslation.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestPathMetadataDynamoDBTranslation.java
index 70d4c3b038..704f51e3c0 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestPathMetadataDynamoDBTranslation.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestPathMetadataDynamoDBTranslation.java
@@ -29,6 +29,7 @@
import com.amazonaws.services.dynamodbv2.model.AttributeDefinition;
import com.amazonaws.services.dynamodbv2.model.KeySchemaElement;
import com.google.common.base.Preconditions;
+import org.junit.After;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Rule;
@@ -114,6 +115,11 @@ public void testKeySchema() {
}
}
+ @After
+ public void tearDown() {
+ PathMetadataDynamoDBTranslation.IGNORED_FIELDS.clear();
+ }
+
@Test
public void testAttributeDefinitions() {
final Collection attrs =
@@ -248,10 +254,11 @@ public void testIsAuthoritativeCompatibilityItemToPathMetadata()
throws Exception {
Item item = Mockito.spy(TEST_DIR_ITEM);
item.withBoolean(IS_AUTHORITATIVE, true);
+ PathMetadataDynamoDBTranslation.IGNORED_FIELDS.add(IS_AUTHORITATIVE);
final String user =
UserGroupInformation.getCurrentUser().getShortUserName();
- DDBPathMetadata meta = itemToPathMetadata(item, user, true);
+ DDBPathMetadata meta = itemToPathMetadata(item, user);
Mockito.verify(item, Mockito.never()).getBoolean(IS_AUTHORITATIVE);
assertFalse(meta.isAuthoritativeDir());
@@ -265,11 +272,48 @@ public void testIsAuthoritativeCompatibilityItemToPathMetadata()
public void testIsAuthoritativeCompatibilityPathMetadataToItem() {
DDBPathMetadata meta = Mockito.spy(testFilePathMetadata);
meta.setAuthoritativeDir(true);
+ PathMetadataDynamoDBTranslation.IGNORED_FIELDS.add(IS_AUTHORITATIVE);
- Item item = pathMetadataToItem(meta, true);
+ Item item = pathMetadataToItem(meta);
Mockito.verify(meta, never()).isAuthoritativeDir();
assertFalse(item.hasAttribute(IS_AUTHORITATIVE));
}
+
+ /**
+ * Test when translating an {@link Item} to {@link DDBPathMetadata} works
+ * if {@code LAST_UPDATED} flag is ignored.
+ */
+ @Test
+ public void testIsLastUpdatedCompatibilityItemToPathMetadata()
+ throws Exception {
+ Item item = Mockito.spy(TEST_DIR_ITEM);
+ item.withLong(LAST_UPDATED, 100);
+ PathMetadataDynamoDBTranslation.IGNORED_FIELDS.add(LAST_UPDATED);
+
+ final String user =
+ UserGroupInformation.getCurrentUser().getShortUserName();
+ DDBPathMetadata meta = itemToPathMetadata(item, user);
+
+ Mockito.verify(item, Mockito.never()).getLong(LAST_UPDATED);
+ assertFalse(meta.isAuthoritativeDir());
+ }
+
+ /**
+ * Test when translating an {@link DDBPathMetadata} to {@link Item} works
+ * if {@code LAST_UPDATED} flag is ignored.
+ */
+ @Test
+ public void testIsLastUpdatedCompatibilityPathMetadataToItem() {
+ DDBPathMetadata meta = Mockito.spy(testFilePathMetadata);
+ meta.setLastUpdated(100);
+ PathMetadataDynamoDBTranslation.IGNORED_FIELDS.add(LAST_UPDATED);
+
+ Item item = pathMetadataToItem(meta);
+
+ Mockito.verify(meta, never()).getLastUpdated();
+ assertFalse(item.hasAttribute(LAST_UPDATED));
+ }
+
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3Guard.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3Guard.java
index 745e7aad28..1ddfed414d 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3Guard.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3Guard.java
@@ -27,6 +27,8 @@
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
+import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_METADATASTORE_AUTHORITATIVE_DIR_TTL;
+
/**
* Tests for the {@link S3Guard} utility class.
*/
@@ -54,8 +56,10 @@ public void testDirListingUnion() throws Exception {
makeFileStatus("s3a://bucket/dir/s3-file4", false)
);
+ S3Guard.ITtlTimeProvider timeProvider = new S3Guard.TtlTimeProvider(
+ DEFAULT_METADATASTORE_AUTHORITATIVE_DIR_TTL);
FileStatus[] result = S3Guard.dirListingUnion(ms, dirPath, s3Listing,
- dirMeta, false);
+ dirMeta, false, timeProvider);
assertEquals("listing length", 4, result.length);
assertContainsPath(result, "s3a://bucket/dir/ms-file1");