big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-13212. RBF: Fix router location cache issue. Contributed by Weiwei Wu.

Browse Source
This commit is contained in:
Inigo Goiri 2018-03-09 17:18:51 -08:00
parent ba0da2785d
commit afe1a3ccd5
2 changed files with 58 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
View File

@ -238,10 +238,18 @@ private void invalidateLocationCache(final String path) {
Entry<String, PathLocation> entry = it.next();
PathLocation loc = entry.getValue();
String src = loc.getSourcePath();
if (src != null) {
if (src.startsWith(path)) {
LOG.debug("Removing {}", src);
it.remove();
}
} else {
String dest = loc.getDefaultLocation().getDest();
if (dest.startsWith(path)) {
LOG.debug("Removing default cache {}", dest);
it.remove();
}
}
}
LOG.debug("Location cache after invalidation: {}", locationCache);
@ -287,6 +295,7 @@ public void refreshEntries(final Collection<MountTable> entries) {
if (!oldEntries.contains(srcPath)) {
// Add node, it does not exist
this.tree.put(srcPath, entry);
invalidateLocationCache(srcPath);
LOG.info("Added new mount point {} to resolver", srcPath);
} else {
// Node exists, check for updates

46
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/resolver/TestMountTableResolver.java
View File

@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.federation.resolver;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE;
import static org.apache.hadoop.hdfs.DFSConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
@ -82,6 +83,7 @@ private void setupMountTable() throws IOException {
Configuration conf = new Configuration();
conf.setInt(
FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE, TEST_MAX_CACHE_SIZE);
conf.setStrings(DFS_ROUTER_DEFAULT_NAMESERVICE, "0");
mountTable = new MountTableResolver(conf);
// Root mount point
@ -479,4 +481,48 @@ public void testCacheCleaning() throws Exception {
long cacheSize = mountTable.getCacheSize();
assertTrue(cacheSize <= TEST_MAX_CACHE_SIZE);
}
@Test
public void testLocationCache() throws Exception {
List<MountTable> entries = new ArrayList<>();
// Add entry and test location cache
Map<String, String> map1 = getMountTableEntry("1", "/testlocationcache");
MountTable entry1 = MountTable.newInstance("/testlocationcache", map1);
entries.add(entry1);
Map<String, String> map2 = getMountTableEntry("2",
"/anothertestlocationcache");
MountTable entry2 = MountTable.newInstance("/anothertestlocationcache",
map2);
entries.add(entry2);
mountTable.refreshEntries(entries);
assertEquals("1->/testlocationcache/",
mountTable.getDestinationForPath("/testlocationcache").toString());
assertEquals("2->/anothertestlocationcache/",
mountTable.getDestinationForPath("/anothertestlocationcache")
.toString());
// Remove the entry1
entries.remove(entry1);
mountTable.refreshEntries(entries);
// Add the default location and test location cache
assertEquals("0->/testlocationcache",
mountTable.getDestinationForPath("/testlocationcache").toString());
// Add the entry again but mount to another ns
Map<String, String> map3 = getMountTableEntry("3", "/testlocationcache");
MountTable entry3 = MountTable.newInstance("/testlocationcache", map3);
entries.add(entry3);
mountTable.refreshEntries(entries);
// Ensure location cache update correctly
assertEquals("3->/testlocationcache/",
mountTable.getDestinationForPath("/testlocationcache").toString());
// Cleanup before exit
mountTable.removeEntry("/testlocationcache");
mountTable.removeEntry("/anothertestlocationcache");
}
}