big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-13253. RBF: Quota management incorrect parent-child relationship judgement. Contributed by Yiqun Lin.

Browse Source
This commit is contained in:
Yiqun Lin 2018-03-13 10:30:20 +08:00
parent 319defafc1
commit 7fab787de7
4 changed files with 47 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableResolver.java
View File

@ -20,6 +20,7 @@
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ROUTER_DEFAULT_NAMESERVICE;
import static org.apache.hadoop.hdfs.DFSConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE;
import static org.apache.hadoop.hdfs.DFSConfigKeys.FEDERATION_MOUNT_TABLE_MAX_CACHE_SIZE_DEFAULT;
import static org.apache.hadoop.hdfs.server.federation.router.FederationUtil.isParentEntry;
import java.io.IOException;
import java.util.Collection;
@ -239,7 +240,7 @@ private void invalidateLocationCache(final String path) {
PathLocation loc = entry.getValue();
String src = loc.getSourcePath();
if (src != null) {
if (src.startsWith(path)) {
if(isParentEntry(src, path)) {
LOG.debug("Removing {}", src);
it.remove();
}
@ -530,17 +531,6 @@ public String getDefaultNamespace() {
return this.defaultNameService;
}
private boolean isParentEntry(final String path, final String parent) {
if (!path.startsWith(parent)) {
return false;
}
if (path.equals(parent)) {
return true;
}
return path.charAt(parent.length()) == Path.SEPARATOR_CHAR
|| parent.equals(Path.SEPARATOR);
}
/**
* Find the deepest mount point for a path.
* @param path Path to look for.

20
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
View File

@ -26,6 +26,7 @@
import java.net.URLConnection;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
@ -186,4 +187,23 @@ public static ActiveNamenodeResolver newActiveNamenodeResolver(
ActiveNamenodeResolver.class);
return newInstance(conf, stateStore, StateStoreService.class, clazz);
}
/**
* Check if the given path is the child of parent path.
* @param path Path to be check.
* @param parent Parent path.
* @return True if parent path is parent entry for given path.
*/
public static boolean isParentEntry(final String path, final String parent) {
if (!path.startsWith(parent)) {
return false;
}
if (path.equals(parent)) {
return true;
}
return path.charAt(parent.length()) == Path.SEPARATOR_CHAR
|| parent.equals(Path.SEPARATOR);
}
}

14
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaManager.java
View File

@ -17,6 +17,9 @@
*/
package org.apache.hadoop.hdfs.server.federation.router;
import static org.apache.hadoop.hdfs.server.federation.router.FederationUtil.isParentEntry;
import java.util.HashSet;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
@ -94,7 +97,16 @@ public Set<String> getPaths(String parentPath) {
String from = parentPath;
String to = parentPath + Character.MAX_VALUE;
SortedMap<String, RouterQuotaUsage> subMap = this.cache.subMap(from, to);
return subMap.keySet();
Set<String> validPaths = new HashSet<>();
if (subMap != null) {
for (String path : subMap.keySet()) {
if (isParentEntry(path, parentPath)) {
validPaths.add(path);
}
}
}
return validPaths;
} finally {
readLock.unlock();
}

12
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuotaManager.java
View File

@ -57,6 +57,18 @@ public void testGetChildrenPaths() {
assertTrue(childrenPaths.contains("/path1/subdir")
&& childrenPaths.contains("/path1/subdir/subdir")
&& childrenPaths.contains("/path1"));
// test for corner case
manager.put("/path3", quotaUsage);
manager.put("/path3/subdir", quotaUsage);
manager.put("/path3-subdir", quotaUsage);
childrenPaths = manager.getPaths("/path3");
assertEquals(2, childrenPaths.size());
// path /path3-subdir should not be returned
assertTrue(childrenPaths.contains("/path3")
&& childrenPaths.contains("/path3/subdir")
&& !childrenPaths.contains("/path3-subdir"));
}
@Test