big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-14955. RBF: getQuotaUsage() on mount point should return global quota. Contributed by Jinglun.

Browse Source
This commit is contained in:
Ayush Saxena 2019-11-18 16:04:52 +05:30
parent 3b5a0e86c1
commit 12617fad2e
3 changed files with 57 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Quota.java
View File

@ -125,7 +125,7 @@ void setQuotaInternal(String path, List<RemoteLocation> locations,
* @throws IOException If the quota system is disabled.
*/
public QuotaUsage getQuotaUsage(String path) throws IOException {
return aggregateQuota(getEachQuotaUsage(path));
return aggregateQuota(path, getEachQuotaUsage(path));
}
/**
@ -234,20 +234,26 @@ private List<RemoteLocation> getValidQuotaLocations(String path)
/**
* Aggregate quota that queried from sub-clusters.
* @param path Federation path of the results.
* @param results Quota query result.
* @return Aggregated Quota.
*/
QuotaUsage aggregateQuota(Map<RemoteLocation, QuotaUsage> results) {
QuotaUsage aggregateQuota(String path,
Map<RemoteLocation, QuotaUsage> results) throws IOException {
long nsCount = 0;
long ssCount = 0;
long nsQuota = HdfsConstants.QUOTA_RESET;
long ssQuota = HdfsConstants.QUOTA_RESET;
boolean hasQuotaUnset = false;
boolean isMountEntry = isMountEntry(path);
for (Map.Entry<RemoteLocation, QuotaUsage> entry : results.entrySet()) {
RemoteLocation loc = entry.getKey();
QuotaUsage usage = entry.getValue();
if (usage != null) {
if (isMountEntry) {
nsCount += usage.getFileAndDirectoryCount();
ssCount += usage.getSpaceConsumed();
} else if (usage != null) {
// If quota is not set in real FileSystem, the usage
// value will return -1.
if (usage.getQuota() == -1 && usage.getSpaceQuota() == -1) {
@ -266,6 +272,11 @@ QuotaUsage aggregateQuota(Map<RemoteLocation, QuotaUsage> results) {
}
}
if (isMountEntry) {
QuotaUsage quota = getGlobalQuota(path);
nsQuota = quota.getQuota();
ssQuota = quota.getSpaceQuota();
}
QuotaUsage.Builder builder = new QuotaUsage.Builder()
.fileAndDirectoryCount(nsCount).spaceConsumed(ssCount);
if (hasQuotaUnset) {

2
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUpdateService.java
View File

@ -110,7 +110,7 @@ protected void periodicInvoke() {
Quota quotaModule = this.rpcServer.getQuotaModule();
Map<RemoteLocation, QuotaUsage> usageMap =
quotaModule.getEachQuotaUsage(src);
currentQuotaUsage = quotaModule.aggregateQuota(usageMap);
currentQuotaUsage = quotaModule.aggregateQuota(src, usageMap);
remoteQuotaUsage.putAll(usageMap);
} catch (IOException ioe) {
LOG.error("Unable to get quota usage for " + src, ioe);

42
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java
View File

@ -964,6 +964,48 @@ public void testFixGlobalQuota() throws Exception {
assertEquals(-1, qu.getSpaceQuota());
}
@Test
public void testGetQuotaUsageOnMountPoint() throws Exception {
long nsQuota = 5;
long ssQuota = 3 * BLOCK_SIZE;
prepareGlobalQuotaTestMountTable(nsQuota, ssQuota);
final FileSystem routerFs = routerContext.getFileSystem();
// Verify getQuotaUsage() of the mount point /dir-1.
QuotaUsage quotaUsage = routerFs.getQuotaUsage(new Path("/dir-1"));
assertEquals(nsQuota, quotaUsage.getQuota());
assertEquals(ssQuota, quotaUsage.getSpaceQuota());
// Verify getQuotaUsage() of the mount point /dir-1/dir-2.
quotaUsage = routerFs.getQuotaUsage(new Path("/dir-1/dir-2"));
assertEquals(nsQuota, quotaUsage.getQuota());
assertEquals(ssQuota * 2, quotaUsage.getSpaceQuota());
// Verify getQuotaUsage() of the mount point /dir-1/dir-2/dir-3
quotaUsage = routerFs.getQuotaUsage(new Path("/dir-1/dir-2/dir-3"));
assertEquals(nsQuota, quotaUsage.getQuota());
assertEquals(ssQuota * 2, quotaUsage.getSpaceQuota());
// Verify getQuotaUsage() of the mount point /dir-4
quotaUsage = routerFs.getQuotaUsage(new Path("/dir-4"));
assertEquals(-1, quotaUsage.getQuota());
assertEquals(-1, quotaUsage.getSpaceQuota());
routerFs.mkdirs(new Path("/dir-1/dir-normal"));
try {
// Verify getQuotaUsage() of the normal path /dir-1/dir-normal.
quotaUsage = routerFs.getQuotaUsage(new Path("/dir-1/dir-normal"));
assertEquals(-1, quotaUsage.getQuota());
assertEquals(-1, quotaUsage.getSpaceQuota());
routerFs.setQuota(new Path("/dir-1/dir-normal"), 100, 200);
// Verify getQuotaUsage() of the normal path /dir-1/dir-normal.
quotaUsage = routerFs.getQuotaUsage(new Path("/dir-1/dir-normal"));
assertEquals(100, quotaUsage.getQuota());
assertEquals(200, quotaUsage.getSpaceQuota());
} finally {
// clear normal path.
routerFs.delete(new Path("/dir-1/dir-normal"), true);
}
}
/**
* Add three mount tables.
* /dir-1 --> ns0---/dir-1 [nsQuota, ssQuota]