big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-13821. RBF: Add dfs.federation.router.mount-table.cache.enable so that users can disable cache. Contributed by Fei Hui.

Browse Source
This commit is contained in:
Yiqun Lin 2018-08-22 11:43:40 +08:00
parent e557c6bd8d
commit 81847392ba
4 changed files with 71 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
View File

@ -22,6 +22,8 @@
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE;
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE;
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE_DEFAULT;
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_CACHE_ENABLE;
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_CACHE_ENABLE_DEFAULT;
import static org.apache.hadoop.hdfs.server.federation.router.FederationUtil.isParentEntry;
import java.io.IOException;
@ -124,12 +126,19 @@ public MountTableResolver(Configuration conf, Router routerService,
this.stateStore = null;
}
int maxCacheSize = conf.getInt(
FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE,
FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE_DEFAULT);
this.locationCache = CacheBuilder.newBuilder()
.maximumSize(maxCacheSize)
.build();
boolean mountTableCacheEnable = conf.getBoolean(
FEDERATION_MOUNT_TABLE_CACHE_ENABLE,
FEDERATION_MOUNT_TABLE_CACHE_ENABLE_DEFAULT);
if (mountTableCacheEnable) {
int maxCacheSize = conf.getInt(
FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE,
FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE_DEFAULT);
this.locationCache = CacheBuilder.newBuilder()
.maximumSize(maxCacheSize)
.build();
} else {
this.locationCache = null;
}
registerCacheExternal();
initDefaultNameService(conf);
@ -239,7 +248,7 @@ public void removeEntry(final String srcPath) {
*/
private void invalidateLocationCache(final String path) {
LOG.debug("Invalidating {} from {}", path, locationCache);
if (locationCache.size() == 0) {
if (locationCache == null || locationCache.size() == 0) {
return;
}
@ -359,7 +368,9 @@ public void clear() {
LOG.info("Clearing all mount location caches");
writeLock.lock();
try {
this.locationCache.invalidateAll();
if (this.locationCache != null) {
this.locationCache.invalidateAll();
}
this.tree.clear();
} finally {
writeLock.unlock();
@ -372,6 +383,9 @@ public PathLocation getDestinationForPath(final String path)
verifyMountTable();
readLock.lock();
try {
if (this.locationCache == null) {
return lookupLocation(path);
}
Callable<? extends PathLocation> meh = new Callable<PathLocation>() {
@Override
public PathLocation call() throws Exception {
@ -603,7 +617,10 @@ private List<MountTable> getTreeValues(final String path, boolean reverse) {
* Get the size of the cache.
* @return Size of the cache.
*/
protected long getCacheSize() {
return this.locationCache.size();
protected long getCacheSize() throws IOException{
if (this.locationCache != null) {
return this.locationCache.size();
}
throw new IOException("localCache is null");
}
}

4
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RBFConfigKeys.java
View File

@ -186,6 +186,10 @@ public class RBFConfigKeys extends CommonConfigurationKeysPublic {
FEDERATION_ROUTER_PREFIX + "mount-table.max-cache-size";
/** Remove cache entries if we have more than 10k. */
public static final int FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE_DEFAULT = 10000;
public static final String FEDERATION_MOUNT_TABLE_CACHE_ENABLE =
FEDERATION_ROUTER_PREFIX + "mount-table.cache.enable";
public static final boolean FEDERATION_MOUNT_TABLE_CACHE_ENABLE_DEFAULT =
true;
// HDFS Router-based federation admin
public static final String DFS_ROUTER_ADMIN_HANDLER_COUNT_KEY =

9
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/resources/hdfs-rbf-default.xml
View File

@ -393,6 +393,15 @@
</description>
</property>
<property>
<name>dfs.federation.router.mount-table.cache.enable</name>
<value>true</value>
<description>
Set to true to enable mount table cache (Path to Remote Location cache).
Disabling the cache is recommended when a large amount of unique paths are queried.
</description>
</property>
<property>
<name>dfs.federation.router.quota.enable</name>
<value>false</value>

31
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
View File

@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.federation.resolver;
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_CACHE_ENABLE;
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE;
import static org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE;
import static org.junit.Assert.assertEquals;
@ -37,6 +38,7 @@
import org.apache.hadoop.hdfs.server.federation.router.Router;
import org.apache.hadoop.hdfs.server.federation.store.MountTableStore;
import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
@ -472,6 +474,35 @@ public void testUpdate() throws IOException {
assertNull(entry2);
}
@Test
public void testDisableLocalCache() throws IOException {
Configuration conf = new Configuration();
// Disable mount table cache
conf.setBoolean(FEDERATION_MOUNT_TABLE_CACHE_ENABLE, false);
conf.setStrings(DFS_ROUTER_DEFAULT_NAMESERVICE, "0");
MountTableResolver tmpMountTable = new MountTableResolver(conf);
// Root mount point
Map<String, String> map = getMountTableEntry("1", "/");
tmpMountTable.addEntry(MountTable.newInstance("/", map));
// /tmp
map = getMountTableEntry("2", "/tmp");
tmpMountTable.addEntry(MountTable.newInstance("/tmp", map));
// Check localCache is null
try {
tmpMountTable.getCacheSize();
fail("getCacheSize call should fail.");
} catch (IOException e) {
GenericTestUtils.assertExceptionContains("localCache is null", e);
}
// Check resolve path without cache
assertEquals("2->/tmp/tesfile1.txt",
tmpMountTable.getDestinationForPath("/tmp/tesfile1.txt").toString());
}
@Test
public void testCacheCleaning() throws Exception {
for (int i = 0; i < 1000; i++) {