big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-15709 Move S3Guard LocalMetadataStore constants to org.apache.hadoop.fs.s3a.Constants (Contributed by Gabor Bota)

Browse Source
This commit is contained in:
Gabor Bota 2018-09-07 10:25:20 -07:00 committed by Aaron Fabbri
parent 94ed5cffd7
commit 36c7c78260
No known key found for this signature in database
GPG Key ID: B2EEFA9E78118A29
2 changed files with 24 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
View File

@ -487,6 +487,24 @@ private Constants() {
public static final String S3GUARD_METASTORE_LOCAL public static final String S3GUARD_METASTORE_LOCAL
= "org.apache.hadoop.fs.s3a.s3guard.LocalMetadataStore"; = "org.apache.hadoop.fs.s3a.s3guard.LocalMetadataStore";
/**
* Maximum number of records in LocalMetadataStore.
*/
@InterfaceStability.Unstable
public static final String S3GUARD_METASTORE_LOCAL_MAX_RECORDS =
"fs.s3a.s3guard.local.max_records";
public static final int DEFAULT_S3GUARD_METASTORE_LOCAL_MAX_RECORDS = 256;
/**
* Time to live in milliseconds in LocalMetadataStore.
* If zero, time-based expiration is disabled.
*/
@InterfaceStability.Unstable
public static final String S3GUARD_METASTORE_LOCAL_ENTRY_TTL =
"fs.s3a.s3guard.local.ttl";
public static final int DEFAULT_S3GUARD_METASTORE_LOCAL_ENTRY_TTL
= 10 * 1000;
/** /**
* Use DynamoDB for the metadata: {@value}. * Use DynamoDB for the metadata: {@value}.
*/ */

26
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
View File

@ -24,7 +24,6 @@
import com.google.common.cache.Cache; import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheBuilder;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
@ -41,6 +40,8 @@
import java.util.Map; import java.util.Map;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import static org.apache.hadoop.fs.s3a.Constants.*;
/** /**
* This is a local, in-memory implementation of MetadataStore. * This is a local, in-memory implementation of MetadataStore.
* This is <i>not</i> a coherent cache across processes. It is only * This is <i>not</i> a coherent cache across processes. It is only
@ -60,23 +61,6 @@
public class LocalMetadataStore implements MetadataStore { public class LocalMetadataStore implements MetadataStore {
public static final Logger LOG = LoggerFactory.getLogger(MetadataStore.class); public static final Logger LOG = LoggerFactory.getLogger(MetadataStore.class);
public static final int DEFAULT_MAX_RECORDS = 256;
public static final int DEFAULT_CACHE_ENTRY_TTL_MSEC = 10 * 1000;
/**
* Maximum number of records.
*/
@InterfaceStability.Evolving
public static final String CONF_MAX_RECORDS =
"fs.metadatastore.local.max_records";
/**
* Time to live in milliseconds. If zero, time-based expiration is
* disabled.
*/
@InterfaceStability.Evolving
public static final String CONF_CACHE_ENTRY_TTL =
"fs.metadatastore.local.ttl";
/** Contains directory and file listings. */ /** Contains directory and file listings. */
private Cache<Path, LocalMetadataEntry> localCache; private Cache<Path, LocalMetadataEntry> localCache;
@ -101,11 +85,13 @@ public void initialize(FileSystem fileSystem) throws IOException {
@Override @Override
public void initialize(Configuration conf) throws IOException { public void initialize(Configuration conf) throws IOException {
Preconditions.checkNotNull(conf); Preconditions.checkNotNull(conf);
int maxRecords = conf.getInt(CONF_MAX_RECORDS, DEFAULT_MAX_RECORDS); int maxRecords = conf.getInt(S3GUARD_METASTORE_LOCAL_MAX_RECORDS,
DEFAULT_S3GUARD_METASTORE_LOCAL_MAX_RECORDS);
if (maxRecords < 4) { if (maxRecords < 4) {
maxRecords = 4; maxRecords = 4;
} }
int ttl = conf.getInt(CONF_CACHE_ENTRY_TTL, DEFAULT_CACHE_ENTRY_TTL_MSEC); int ttl = conf.getInt(S3GUARD_METASTORE_LOCAL_ENTRY_TTL,
DEFAULT_S3GUARD_METASTORE_LOCAL_ENTRY_TTL);
CacheBuilder builder = CacheBuilder.newBuilder().maximumSize(maxRecords); CacheBuilder builder = CacheBuilder.newBuilder().maximumSize(maxRecords);
if (ttl >= 0) { if (ttl >= 0) {