big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-15510. RBF: Quota and Content Summary was not correct in Multiple Destinations. Contributed by Hemanth Boyina.

Browse Source 
(cherry picked from commit ca8e7a7725)
This commit is contained in:
Takanobu Asanuma 2020-08-27 12:10:39 +09:00 committed by Wei-Chiu Chuang
parent a549b4a82e
commit f5cc1540b4
2 changed files with 61 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java
View File

@ -1857,6 +1857,8 @@ private RemoteLocation getFirstMatchingLocation(RemoteLocation location,
/** /**
* Aggregate content summaries for each subcluster. * Aggregate content summaries for each subcluster.
* If the mount point has multiple destinations
* add the quota set value only once.
* *
* @param summaries Collection of individual summaries. * @param summaries Collection of individual summaries.
* @return Aggregated content summary. * @return Aggregated content summary.
@ -1879,9 +1881,9 @@ private ContentSummary aggregateContentSummary(
length += summary.getLength(); length += summary.getLength();
fileCount += summary.getFileCount(); fileCount += summary.getFileCount();
directoryCount += summary.getDirectoryCount(); directoryCount += summary.getDirectoryCount();
quota += summary.getQuota(); quota = summary.getQuota();
spaceConsumed += summary.getSpaceConsumed(); spaceConsumed += summary.getSpaceConsumed();
spaceQuota += summary.getSpaceQuota(); spaceQuota = summary.getSpaceQuota();
// We return from the first response as we assume that the EC policy // We return from the first response as we assume that the EC policy
// of each sub-cluster is same. // of each sub-cluster is same.
if (ecPolicy.isEmpty()) { if (ecPolicy.isEmpty()) {

57
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java
View File

@ -550,6 +550,63 @@ public void testClearQuota() throws Exception {
assertEquals(-1, cs1.getSpaceQuota()); assertEquals(-1, cs1.getSpaceQuota());
} }
@Test
public void testContentSummaryWithMultipleDest() throws Exception {
MountTable addEntry;
long nsQuota = 5;
long ssQuota = 100;
Path path = new Path("/testContentSummaryWithMultipleDest");
Map<String, String> destMap = new HashMap<>();
destMap.put("ns0", "/testContentSummaryWithMultipleDest");
destMap.put("ns1", "/testContentSummaryWithMultipleDest");
nnFs0.mkdirs(path);
nnFs1.mkdirs(path);
addEntry =
MountTable.newInstance("/testContentSummaryWithMultipleDest", destMap);
addEntry.setQuota(
new RouterQuotaUsage.Builder().quota(nsQuota).spaceQuota(ssQuota)
.build());
assertTrue(addMountTable(addEntry));
RouterQuotaUpdateService updateService =
routerContext.getRouter().getQuotaCacheUpdateService();
updateService.periodicInvoke();
ContentSummary cs = routerFs.getContentSummary(path);
assertEquals(nsQuota, cs.getQuota());
assertEquals(ssQuota, cs.getSpaceQuota());
ContentSummary ns0Cs = nnFs0.getContentSummary(path);
assertEquals(nsQuota, ns0Cs.getQuota());
assertEquals(ssQuota, ns0Cs.getSpaceQuota());
ContentSummary ns1Cs = nnFs1.getContentSummary(path);
assertEquals(nsQuota, ns1Cs.getQuota());
assertEquals(ssQuota, ns1Cs.getSpaceQuota());
}
@Test
public void testContentSummaryMultipleDestWithMaxValue()
throws Exception {
MountTable addEntry;
long nsQuota = Long.MAX_VALUE - 2;
long ssQuota = Long.MAX_VALUE - 2;
Path path = new Path("/testContentSummaryMultipleDestWithMaxValue");
Map<String, String> destMap = new HashMap<>();
destMap.put("ns0", "/testContentSummaryMultipleDestWithMaxValue");
destMap.put("ns1", "/testContentSummaryMultipleDestWithMaxValue");
nnFs0.mkdirs(path);
nnFs1.mkdirs(path);
addEntry = MountTable
.newInstance("/testContentSummaryMultipleDestWithMaxValue", destMap);
addEntry.setQuota(
new RouterQuotaUsage.Builder().quota(nsQuota).spaceQuota(ssQuota)
.build());
assertTrue(addMountTable(addEntry));
RouterQuotaUpdateService updateService =
routerContext.getRouter().getQuotaCacheUpdateService();
updateService.periodicInvoke();
ContentSummary cs = routerFs.getContentSummary(path);
assertEquals(nsQuota, cs.getQuota());
assertEquals(ssQuota, cs.getSpaceQuota());
}
/** /**
* Test to verify rename operation on directories in case of multiple * Test to verify rename operation on directories in case of multiple
* destinations. * destinations.