big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-13967. HDFS Router Quota Class Review. Contributed by BELUGA BEHR.

Browse Source
This commit is contained in:
Yiqun Lin 2018-10-09 16:11:07 +08:00
parent 9bbeb52486
commit d4626b4d18
1 changed files with 28 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java
View File

@ -18,12 +18,14 @@
package org.apache.hadoop.hdfs.server.federation.router; package org.apache.hadoop.hdfs.server.federation.router;
import java.io.IOException; import java.io.IOException;
import java.util.HashMap; import java.util.ArrayList;
import java.util.LinkedList; import java.util.Collection;
import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.QuotaUsage; import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.ClientProtocol;
@ -33,6 +35,9 @@
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ListMultimap;
/** /**
* Module that implements the quota relevant RPC calls * Module that implements the quota relevant RPC calls
* {@link ClientProtocol#setQuota(String, long, long, StorageType)} * {@link ClientProtocol#setQuota(String, long, long, StorageType)}
@ -121,37 +126,31 @@ private List<RemoteLocation> getValidQuotaLocations(String path)
final List<RemoteLocation> locations = getQuotaRemoteLocations(path); final List<RemoteLocation> locations = getQuotaRemoteLocations(path);
// NameService -> Locations // NameService -> Locations
Map<String, List<RemoteLocation>> validLocations = new HashMap<>(); ListMultimap<String, RemoteLocation> validLocations =
for (RemoteLocation loc : locations) { ArrayListMultimap.create();
String nsId = loc.getNameserviceId();
List<RemoteLocation> dests = validLocations.get(nsId);
if (dests == null) {
dests = new LinkedList<>();
dests.add(loc);
validLocations.put(nsId, dests);
} else {
// Ensure the paths in the same nameservice is different.
// Don't include parent-child paths.
boolean isChildPath = false;
for (RemoteLocation d : dests) {
if (loc.getDest().startsWith(d.getDest())) {
isChildPath = true;
break;
}
}
if (!isChildPath) { for (RemoteLocation loc : locations) {
dests.add(loc); final String nsId = loc.getNameserviceId();
final Collection<RemoteLocation> dests = validLocations.get(nsId);
// Ensure the paths in the same nameservice is different.
// Do not include parent-child paths.
boolean isChildPath = false;
for (RemoteLocation d : dests) {
if (StringUtils.startsWith(loc.getDest(), d.getDest())) {
isChildPath = true;
break;
} }
} }
if (!isChildPath) {
validLocations.put(nsId, loc);
}
} }
List<RemoteLocation> quotaLocs = new LinkedList<>(); return Collections
for (List<RemoteLocation> locs : validLocations.values()) { .unmodifiableList(new ArrayList<>(validLocations.values()));
quotaLocs.addAll(locs);
}
return quotaLocs;
} }
/** /**
@ -209,7 +208,7 @@ private QuotaUsage aggregateQuota(Map<RemoteLocation, QuotaUsage> results) {
*/ */
private List<RemoteLocation> getQuotaRemoteLocations(String path) private List<RemoteLocation> getQuotaRemoteLocations(String path)
throws IOException { throws IOException {
List<RemoteLocation> locations = new LinkedList<>(); List<RemoteLocation> locations = new ArrayList<>();
RouterQuotaManager manager = this.router.getQuotaManager(); RouterQuotaManager manager = this.router.getQuotaManager();
if (manager != null) { if (manager != null) {
Set<String> childrenPaths = manager.getPaths(path); Set<String> childrenPaths = manager.getPaths(path);
@ -217,7 +216,6 @@ private List<RemoteLocation> getQuotaRemoteLocations(String path)
locations.addAll(rpcServer.getLocationsForPath(childPath, true, false)); locations.addAll(rpcServer.getLocationsForPath(childPath, true, false));
} }
} }
return locations; return locations;
} }
} }