big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-14499. Misleading REM_QUOTA value with snapshot and trash feature enabled for a directory. Contributed by Shashikant Banerjee.

Browse Source
This commit is contained in:
Shashikant Banerjee 2019-07-12 15:41:34 +05:30
parent 9119ed07ff
commit f9fab9f22a
2 changed files with 34 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java
View File

@ -500,14 +500,15 @@ public int getLastSnapshotId() {
@Override @Override
public final ContentSummaryComputationContext computeContentSummary( public final ContentSummaryComputationContext computeContentSummary(
int snapshotId, ContentSummaryComputationContext summary) { int snapshotId, ContentSummaryComputationContext summary)
final int s = snapshotId < lastSnapshotId ? snapshotId : lastSnapshotId; throws AccessControlException {
// only count storagespace for WithName Preconditions.checkState(snapshotId == Snapshot.CURRENT_STATE_ID
final QuotaCounts q = computeQuotaUsage( || this.lastSnapshotId >= snapshotId);
summary.getBlockStoragePolicySuite(), getStoragePolicyID(), false, s); final INode referred =
summary.getCounts().addContent(Content.DISKSPACE, q.getStorageSpace()); this.getReferredINode().asReference().getReferredINode();
summary.getCounts().addTypeSpaces(q.getTypeSpaces()); int id = snapshotId != Snapshot.CURRENT_STATE_ID ? snapshotId :
return summary; this.lastSnapshotId;
return referred.computeContentSummary(id, summary);
} }
@Override @Override

33
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestGetContentSummaryWithSnapshot.java
View File

@ -90,18 +90,22 @@ public void testGetContentSummary() throws IOException {
final Path foo = new Path("/foo"); final Path foo = new Path("/foo");
final Path bar = new Path(foo, "bar"); final Path bar = new Path(foo, "bar");
final Path baz = new Path(bar, "baz"); final Path baz = new Path(bar, "baz");
final Path qux = new Path(bar, "qux");
final Path temp = new Path("/temp");
dfs.mkdirs(bar); dfs.mkdirs(bar);
dfs.mkdirs(temp);
dfs.allowSnapshot(foo); dfs.allowSnapshot(foo);
dfs.createSnapshot(foo, "s1"); dfs.createSnapshot(foo, "s1");
DFSTestUtil.createFile(dfs, baz, 10, REPLICATION, 0L); DFSTestUtil.createFile(dfs, baz, 10, REPLICATION, 0L);
DFSTestUtil.createFile(dfs, qux, 10, REPLICATION, 0L);
ContentSummary summary = cluster.getNameNodeRpc().getContentSummary( ContentSummary summary = cluster.getNameNodeRpc().getContentSummary(
bar.toString()); bar.toString());
Assert.assertEquals(1, summary.getDirectoryCount()); Assert.assertEquals(1, summary.getDirectoryCount());
Assert.assertEquals(1, summary.getFileCount()); Assert.assertEquals(2, summary.getFileCount());
Assert.assertEquals(10, summary.getLength()); Assert.assertEquals(20, summary.getLength());
final Path barS1 = SnapshotTestHelper.getSnapshotPath(foo, "s1", "bar"); final Path barS1 = SnapshotTestHelper.getSnapshotPath(foo, "s1", "bar");
summary = cluster.getNameNodeRpc().getContentSummary(barS1.toString()); summary = cluster.getNameNodeRpc().getContentSummary(barS1.toString());
@ -112,8 +116,8 @@ public void testGetContentSummary() throws IOException {
// also check /foo and /foo/.snapshot/s1 // also check /foo and /foo/.snapshot/s1
summary = cluster.getNameNodeRpc().getContentSummary(foo.toString()); summary = cluster.getNameNodeRpc().getContentSummary(foo.toString());
Assert.assertEquals(2, summary.getDirectoryCount()); Assert.assertEquals(2, summary.getDirectoryCount());
Assert.assertEquals(1, summary.getFileCount()); Assert.assertEquals(2, summary.getFileCount());
Assert.assertEquals(10, summary.getLength()); Assert.assertEquals(20, summary.getLength());
final Path fooS1 = SnapshotTestHelper.getSnapshotRoot(foo, "s1"); final Path fooS1 = SnapshotTestHelper.getSnapshotRoot(foo, "s1");
summary = cluster.getNameNodeRpc().getContentSummary(fooS1.toString()); summary = cluster.getNameNodeRpc().getContentSummary(fooS1.toString());
@ -127,14 +131,14 @@ public void testGetContentSummary() throws IOException {
summary = cluster.getNameNodeRpc().getContentSummary( summary = cluster.getNameNodeRpc().getContentSummary(
bar.toString()); bar.toString());
Assert.assertEquals(1, summary.getDirectoryCount()); Assert.assertEquals(1, summary.getDirectoryCount());
Assert.assertEquals(1, summary.getFileCount()); Assert.assertEquals(2, summary.getFileCount());
Assert.assertEquals(20, summary.getLength()); Assert.assertEquals(30, summary.getLength());
final Path fooS2 = SnapshotTestHelper.getSnapshotRoot(foo, "s2"); final Path fooS2 = SnapshotTestHelper.getSnapshotRoot(foo, "s2");
summary = cluster.getNameNodeRpc().getContentSummary(fooS2.toString()); summary = cluster.getNameNodeRpc().getContentSummary(fooS2.toString());
Assert.assertEquals(2, summary.getDirectoryCount()); Assert.assertEquals(2, summary.getDirectoryCount());
Assert.assertEquals(1, summary.getFileCount()); Assert.assertEquals(2, summary.getFileCount());
Assert.assertEquals(10, summary.getLength()); Assert.assertEquals(20, summary.getLength());
cluster.getNameNodeRpc().delete(baz.toString(), false); cluster.getNameNodeRpc().delete(baz.toString(), false);
@ -143,11 +147,24 @@ public void testGetContentSummary() throws IOException {
Assert.assertEquals(0, summary.getSnapshotDirectoryCount()); Assert.assertEquals(0, summary.getSnapshotDirectoryCount());
Assert.assertEquals(1, summary.getSnapshotFileCount()); Assert.assertEquals(1, summary.getSnapshotFileCount());
Assert.assertEquals(20, summary.getSnapshotLength()); Assert.assertEquals(20, summary.getSnapshotLength());
Assert.assertEquals(2, summary.getDirectoryCount());
Assert.assertEquals(2, summary.getFileCount());
Assert.assertEquals(30, summary.getLength());
final Path bazS1 = SnapshotTestHelper.getSnapshotPath(foo, "s1", "bar/baz"); final Path bazS1 = SnapshotTestHelper.getSnapshotPath(foo, "s1", "bar/baz");
try { try {
cluster.getNameNodeRpc().getContentSummary(bazS1.toString()); cluster.getNameNodeRpc().getContentSummary(bazS1.toString());
Assert.fail("should get FileNotFoundException"); Assert.fail("should get FileNotFoundException");
} catch (FileNotFoundException ignored) {} } catch (FileNotFoundException ignored) {}
cluster.getNameNodeRpc().rename(qux.toString(), "/temp/qux");
summary = cluster.getNameNodeRpc().getContentSummary(
foo.toString());
Assert.assertEquals(0, summary.getSnapshotDirectoryCount());
Assert.assertEquals(2, summary.getSnapshotFileCount());
Assert.assertEquals(30, summary.getSnapshotLength());
Assert.assertEquals(2, summary.getDirectoryCount());
Assert.assertEquals(2, summary.getFileCount());
Assert.assertEquals(30, summary.getLength());
} }
} }