big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-14957. INodeReference Space Consumed was not same in QuotaUsage and ContentSummary. Contributed by hemanthboyina.

Browse Source
This commit is contained in:
Surendra Singh Lilhore 2020-01-09 12:04:05 +05:30
parent 8fe01db34a
commit bf45f3b80a
2 changed files with 65 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
View File

@ -873,8 +873,18 @@ public final ContentSummaryComputationContext computeContentSummary(
counts.addContent(Content.FILE, 1);
final long fileLen = computeFileSize(snapshotId);
counts.addContent(Content.LENGTH, fileLen);
counts.addContent(Content.DISKSPACE, storagespaceConsumed(null)
.getStorageSpace());
FileWithSnapshotFeature sf = getFileWithSnapshotFeature();
if (sf == null) {
counts.addContent(Content.DISKSPACE,
storagespaceConsumed(null).getStorageSpace());
} else if (isStriped()) {
counts.addContent(Content.DISKSPACE,
storagespaceConsumedStriped().getStorageSpace());
} else {
long diskSpaceQuota = getDiskSpaceQuota(counts, sf, snapshotId);
counts.addContent(Content.DISKSPACE, diskSpaceQuota);
}
if (getStoragePolicyID() != BLOCK_STORAGE_POLICY_ID_UNSPECIFIED){
BlockStoragePolicy bsp = summary.getBlockStoragePolicySuite().
@ -890,6 +900,34 @@ public final ContentSummaryComputationContext computeContentSummary(
return summary;
}
/**
* Compute disk space consumed by all the blocks in snapshots.
*/
private long getDiskSpaceQuota(ContentCounts counts,
FileWithSnapshotFeature sf, int lastSnapshotId) {
FileDiffList fileDiffList = sf.getDiffs();
int last = fileDiffList.getLastSnapshotId();
if (lastSnapshotId == Snapshot.CURRENT_STATE_ID
|| last == Snapshot.CURRENT_STATE_ID) {
return storagespaceConsumed(null).getStorageSpace();
}
final long ssDeltaNoReplication;
short replication;
if (last < lastSnapshotId) {
ssDeltaNoReplication = computeFileSize(true, false);
replication = getFileReplication();
} else {
int sid = fileDiffList.getSnapshotById(lastSnapshotId);
ssDeltaNoReplication = computeFileSize(sid);
replication = getFileReplication(sid);
}
return ssDeltaNoReplication * replication;
}
/** The same as computeFileSize(null). */
public final long computeFileSize() {
return computeFileSize(CURRENT_STATE_ID);

24
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
View File

@ -2476,4 +2476,28 @@ String printTree(String label) throws Exception {
output.println(b);
return b;
}
/**
* Test getContentsummary and getQuotausage for an INodeReference.
*/
@Test(timeout = 300000)
public void testQuotaForRenameFileInSnapshot() throws Exception {
final Path snapshotDir = new Path("/testRenameWithSnapshot");
hdfs.mkdirs(snapshotDir, new FsPermission((short) 0777));
final Path file = new Path(snapshotDir, "file");
DFSTestUtil.createFile(hdfs, file, BLOCKSIZE, REPL, SEED);
hdfs.allowSnapshot(snapshotDir);
hdfs.createSnapshot(snapshotDir, "s0");
hdfs.mkdirs(new Path("/dir1"));
// Truncate a file which exists in snapshot , that is an
// INodeReference
hdfs.truncate(file, 10);
hdfs.rename(file, new Path("/dir1"));
assertEquals(hdfs.getContentSummary(new Path("/")).getSpaceConsumed(),
hdfs.getQuotaUsage(new Path("/")).getSpaceConsumed());
assertEquals(
hdfs.getContentSummary(new Path("/")).getFileAndDirectoryCount(),
hdfs.getQuotaUsage(new Path("/")).getFileAndDirectoryCount());
}
}