big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-15225. RBF: Add snapshot counts to content summary in router. (#4356). Contributed by Ayush Saxena.

Browse Source 
Reviewed-by: Inigo Goiri <inigoiri@apache.org>
This commit is contained in:
Ayush Saxena 2022-05-27 09:06:07 +05:30 committed by GitHub
parent bee538f785
commit 96985f4c45
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 49 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java
View File

@ -1903,6 +1903,8 @@ private ContentSummary aggregateContentSummary(
long quota = 0; long quota = 0;
long spaceConsumed = 0; long spaceConsumed = 0;
long spaceQuota = 0; long spaceQuota = 0;
long snapshotDirectoryCount = 0;
long snapshotFileCount = 0;
String ecPolicy = ""; String ecPolicy = "";
for (ContentSummary summary : summaries) { for (ContentSummary summary : summaries) {
@ -1912,6 +1914,8 @@ private ContentSummary aggregateContentSummary(
quota = summary.getQuota(); quota = summary.getQuota();
spaceConsumed += summary.getSpaceConsumed(); spaceConsumed += summary.getSpaceConsumed();
spaceQuota = summary.getSpaceQuota(); spaceQuota = summary.getSpaceQuota();
snapshotDirectoryCount += summary.getSnapshotDirectoryCount();
snapshotFileCount += summary.getSnapshotFileCount();
// We return from the first response as we assume that the EC policy // We return from the first response as we assume that the EC policy
// of each sub-cluster is same. // of each sub-cluster is same.
if (ecPolicy.isEmpty()) { if (ecPolicy.isEmpty()) {
@ -1927,6 +1931,8 @@ private ContentSummary aggregateContentSummary(
.spaceConsumed(spaceConsumed) .spaceConsumed(spaceConsumed)
.spaceQuota(spaceQuota) .spaceQuota(spaceQuota)
.erasureCodingPolicy(ecPolicy) .erasureCodingPolicy(ecPolicy)
.snapshotDirectoryCount(snapshotDirectoryCount)
.snapshotFileCount(snapshotFileCount)
.build(); .build();
return ret; return ret;
} }

43
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
View File

@ -56,6 +56,7 @@
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.CryptoProtocolVersion; import org.apache.hadoop.crypto.CryptoProtocolVersion;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileStatus;
@ -2004,4 +2005,46 @@ public void testAddClientIpPortToCallerContext() throws IOException {
assertFalse(auditLog.getOutput().contains("clientIp:1.1.1.1")); assertFalse(auditLog.getOutput().contains("clientIp:1.1.1.1"));
assertFalse(auditLog.getOutput().contains("clientPort:1234")); assertFalse(auditLog.getOutput().contains("clientPort:1234"));
} }
@Test
public void testContentSummaryWithSnapshot() throws Exception {
DistributedFileSystem routerDFS = (DistributedFileSystem) routerFS;
Path dirPath = new Path("/testdir");
Path subdirPath = new Path(dirPath, "subdir");
Path filePath1 = new Path(dirPath, "file");
Path filePath2 = new Path(subdirPath, "file2");
// Create directories.
routerDFS.mkdirs(dirPath);
routerDFS.mkdirs(subdirPath);
// Create files.
createFile(routerDFS, filePath1.toString(), 32);
createFile(routerDFS, filePath2.toString(), 16);
// Allow & Create snapshot.
routerDFS.allowSnapshot(dirPath);
routerDFS.createSnapshot(dirPath, "s1");
try {
// Check content summary, snapshot count should be 0
ContentSummary contentSummary = routerDFS.getContentSummary(dirPath);
assertEquals(0, contentSummary.getSnapshotDirectoryCount());
assertEquals(0, contentSummary.getSnapshotFileCount());
// Delete the file & subdir(Total 2 files deleted & 1 directory)
routerDFS.delete(filePath1, true);
routerDFS.delete(subdirPath, true);
// Get the Content Summary
contentSummary = routerDFS.getContentSummary(dirPath);
assertEquals(1, contentSummary.getSnapshotDirectoryCount());
assertEquals(2, contentSummary.getSnapshotFileCount());
} finally {
// Cleanup
routerDFS.deleteSnapshot(dirPath, "s1");
routerDFS.disallowSnapshot(dirPath);
routerDFS.delete(dirPath, true);
}
}
} }