big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-15214. WebHDFS: Add snapshot counts to Content Summary. Contributed by hemanthboyina.

Browse Source
This commit is contained in:
Takanobu Asanuma 2020-03-22 01:44:55 +09:00
parent 3612317038
commit b74e47e31a
4 changed files with 67 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
View File

@ -440,6 +440,25 @@ static ContentSummary toContentSummary(final Map<?, ?> json) {
.directoryCount(directoryCount)
.erasureCodingPolicy(ecPolicy);
builder = buildQuotaUsage(builder, m, ContentSummary.Builder.class);
if (m.get("snapshotLength") != null) {
long snapshotLength = ((Number) m.get("snapshotLength")).longValue();
builder.snapshotLength(snapshotLength);
}
if (m.get("snapshotFileCount") != null) {
long snapshotFileCount =
((Number) m.get("snapshotFileCount")).longValue();
builder.snapshotFileCount(snapshotFileCount);
}
if (m.get("snapshotDirectoryCount") != null) {
long snapshotDirectoryCount =
((Number) m.get("snapshotDirectoryCount")).longValue();
builder.snapshotDirectoryCount(snapshotDirectoryCount);
}
if (m.get("snapshotSpaceConsumed") != null) {
long snapshotSpaceConsumed =
((Number) m.get("snapshotSpaceConsumed")).longValue();
builder.snapshotSpaceConsumed(snapshotSpaceConsumed);
}
return builder.build();
}

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
View File

@ -358,6 +358,11 @@ public static String toJsonString(final ContentSummary contentsummary) {
// For ContentSummary we don't need this since we already have
// separate count for file and directory.
m.putAll(toJsonMap(contentsummary, false));
m.put("snapshotLength", contentsummary.getSnapshotLength());
m.put("snapshotFileCount", contentsummary.getSnapshotFileCount());
m.put("snapshotDirectoryCount",
contentsummary.getSnapshotDirectoryCount());
m.put("snapshotSpaceConsumed", contentsummary.getSnapshotSpaceConsumed());
return toJsonString(ContentSummary.class, m);
}

23
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java
View File

@ -306,7 +306,13 @@ public void testToJsonFromAclStatus() {
@Test
public void testToJsonFromContentSummary() {
String jsonString =
"{\"ContentSummary\":{\"directoryCount\":33333,\"ecPolicy\":\"RS-6-3-1024k\",\"fileCount\":22222,\"length\":11111,\"quota\":44444,\"spaceConsumed\":55555,\"spaceQuota\":66666,\"typeQuota\":{}}}";
"{\"ContentSummary\":{\"directoryCount\":33333,\"ecPolicy\":"
+ "\"RS-6-3-1024k\",\"fileCount\":22222,\"length\":11111,"
+ "\"quota\":44444,\"snapshotDirectoryCount\":1,"
+ "\"snapshotFileCount\":2,\"snapshotLength\":10,"
+ "\"snapshotSpaceConsumed\":30,\"spaceConsumed\":55555,"
+ "\"spaceQuota\":66666,\"typeQuota\":{}}}";
long length = 11111;
long fileCount = 22222;
long directoryCount = 33333;
@ -314,11 +320,18 @@ public void testToJsonFromContentSummary() {
long spaceConsumed = 55555;
long spaceQuota = 66666;
String ecPolicy = "RS-6-3-1024k";
long snapshotLength = 10;
long snapshotFileCount = 2;
long snapshotDirectoryCount = 1;
long snapshotSpaceConsumed = 30;
ContentSummary contentSummary = new ContentSummary.Builder().length(length).
fileCount(fileCount).directoryCount(directoryCount).quota(quota).
spaceConsumed(spaceConsumed).spaceQuota(spaceQuota).
erasureCodingPolicy(ecPolicy).build();
ContentSummary contentSummary = new ContentSummary.Builder().length(length)
.fileCount(fileCount).directoryCount(directoryCount).quota(quota)
.spaceConsumed(spaceConsumed).spaceQuota(spaceQuota)
.erasureCodingPolicy(ecPolicy).snapshotLength(snapshotLength)
.snapshotFileCount(snapshotFileCount)
.snapshotDirectoryCount(snapshotDirectoryCount)
.snapshotSpaceConsumed(snapshotSpaceConsumed).build();
Assert.assertEquals(jsonString, JsonUtil.toJsonString(contentSummary));
}

24
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
View File

@ -994,6 +994,30 @@ public void testContentSummary() throws Exception {
.assertTrue((contentSummary.getTypeQuota(StorageType.DISK) == 100000));
}
/**
* Test Snapshot related information in ContentSummary.
*/
@Test
public void testSnapshotInContentSummary() throws Exception {
final Configuration conf = WebHdfsTestUtil.createConf();
Path dirPath = new Path("/dir");
final Path filePath = new Path("/dir/file");
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
final WebHdfsFileSystem webHdfs = WebHdfsTestUtil.getWebHdfsFileSystem(conf,
WebHdfsConstants.WEBHDFS_SCHEME);
final DistributedFileSystem dfs = cluster.getFileSystem();
DFSTestUtil.createFile(dfs, filePath, 10, (short) 3, 0L);
dfs.allowSnapshot(dirPath);
dfs.createSnapshot(dirPath);
dfs.delete(filePath, true);
ContentSummary contentSummary = webHdfs.getContentSummary(dirPath);
assertEquals(1, contentSummary.getSnapshotFileCount());
assertEquals(10, contentSummary.getSnapshotLength());
assertEquals(30, contentSummary.getSnapshotSpaceConsumed());
assertEquals(dfs.getContentSummary(dirPath),
webHdfs.getContentSummary(dirPath));
}
@Test
public void testQuotaUsage() throws Exception {
final Configuration conf = WebHdfsTestUtil.createConf();