diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index dae38c218c..c01326be28 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -679,6 +679,9 @@ Release 2.5.0 - UNRELEASED HDFS-6549. Add support for accessing the NFS gateway from the AIX NFS client. (atm) + HDFS-6535. HDFS quota update is wrong when file is appended. (George Wong + via jing9) + BREAKDOWN OF HDFS-2006 SUBTASKS AND RELATED JIRAS HDFS-6299. Protobuf for XAttr and client-side implementation. (Yi Liu via umamahesh) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index ad7b4d6128..af436182ab 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -2512,7 +2512,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, if (ret != null) { // update the quota: use the preferred block size for UC block final long diff = file.getPreferredBlockSize() - ret.getBlockSize(); - dir.updateSpaceConsumed(src, 0, diff); + dir.updateSpaceConsumed(src, 0, diff * file.getBlockReplication()); } if (writeToEditLog) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java index 1c770b141a..fad88070ad 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java @@ -18,10 +18,12 @@ package org.apache.hadoop.hdfs.server.namenode; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import java.util.EnumSet; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; @@ -36,7 +38,9 @@ import org.junit.Test; public class TestDiskspaceQuotaUpdate { private static final int BLOCKSIZE = 1024; - private static final short REPLICATION = 1; + private static final short REPLICATION = 4; + static final long seed = 0L; + private static final Path dir = new Path("/TestQuotaUpdate"); private Configuration conf; private MiniDFSCluster cluster; @@ -62,42 +66,84 @@ public class TestDiskspaceQuotaUpdate { } } + /** + * Test if the quota can be correctly updated for create file + */ + @Test (timeout=60000) + public void testQuotaUpdateWithFileCreate() throws Exception { + final Path foo = new Path(dir, "foo"); + Path createdFile = new Path(foo, "created_file.data"); + dfs.mkdirs(foo); + dfs.setQuota(foo, Long.MAX_VALUE-1, Long.MAX_VALUE-1); + long fileLen = BLOCKSIZE * 2 + BLOCKSIZE / 2; + DFSTestUtil.createFile(dfs, createdFile, BLOCKSIZE / 16, + fileLen, BLOCKSIZE, REPLICATION, seed); + INode fnode = fsdir.getINode4Write(foo.toString()); + assertTrue(fnode.isDirectory()); + assertTrue(fnode.isQuotaSet()); + Quota.Counts cnt = fnode.asDirectory().getDirectoryWithQuotaFeature() + .getSpaceConsumed(); + assertEquals(2, cnt.get(Quota.NAMESPACE)); + assertEquals(fileLen * REPLICATION, cnt.get(Quota.DISKSPACE)); + } + /** * Test if the quota can be correctly updated for append */ - @Test + @Test (timeout=60000) public void testUpdateQuotaForAppend() throws Exception { - final Path foo = new Path("/foo"); + final Path foo = new Path(dir ,"foo"); final Path bar = new Path(foo, "bar"); - DFSTestUtil.createFile(dfs, bar, BLOCKSIZE, REPLICATION, 0L); + long currentFileLen = BLOCKSIZE; + DFSTestUtil.createFile(dfs, bar, currentFileLen, REPLICATION, seed); dfs.setQuota(foo, Long.MAX_VALUE-1, Long.MAX_VALUE-1); - // append half of the block data + // append half of the block data, the previous file length is at block + // boundary DFSTestUtil.appendFile(dfs, bar, BLOCKSIZE / 2); + currentFileLen += (BLOCKSIZE / 2); INodeDirectory fooNode = fsdir.getINode4Write(foo.toString()).asDirectory(); + assertTrue(fooNode.isQuotaSet()); Quota.Counts quota = fooNode.getDirectoryWithQuotaFeature() .getSpaceConsumed(); long ns = quota.get(Quota.NAMESPACE); long ds = quota.get(Quota.DISKSPACE); assertEquals(2, ns); // foo and bar - assertEquals((BLOCKSIZE + BLOCKSIZE / 2) * REPLICATION, ds); + assertEquals(currentFileLen * REPLICATION, ds); + ContentSummary c = dfs.getContentSummary(foo); + assertEquals(c.getSpaceConsumed(), ds); - // append another block + // append another block, the previous file length is not at block boundary DFSTestUtil.appendFile(dfs, bar, BLOCKSIZE); + currentFileLen += BLOCKSIZE; quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed(); ns = quota.get(Quota.NAMESPACE); ds = quota.get(Quota.DISKSPACE); assertEquals(2, ns); // foo and bar - assertEquals((BLOCKSIZE * 2 + BLOCKSIZE / 2) * REPLICATION, ds); + assertEquals(currentFileLen * REPLICATION, ds); + c = dfs.getContentSummary(foo); + assertEquals(c.getSpaceConsumed(), ds); + + // append several blocks + DFSTestUtil.appendFile(dfs, bar, BLOCKSIZE * 3 + BLOCKSIZE / 8); + currentFileLen += (BLOCKSIZE * 3 + BLOCKSIZE / 8); + + quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed(); + ns = quota.get(Quota.NAMESPACE); + ds = quota.get(Quota.DISKSPACE); + assertEquals(2, ns); // foo and bar + assertEquals(currentFileLen * REPLICATION, ds); + c = dfs.getContentSummary(foo); + assertEquals(c.getSpaceConsumed(), ds); } /** * Test if the quota can be correctly updated when file length is updated * through fsync */ - @Test + @Test (timeout=60000) public void testUpdateQuotaForFSync() throws Exception { final Path foo = new Path("/foo"); final Path bar = new Path(foo, "bar");