diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java index 408b3256df..7f053380f8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java @@ -215,6 +215,7 @@ class CellBuffers { buffers = new ByteBuffer[numAllBlocks]; for (int i = 0; i < buffers.length; i++) { buffers[i] = BUFFER_POOL.getBuffer(useDirectBuffer(), cellSize); + buffers[i].limit(cellSize); } } @@ -237,6 +238,7 @@ private int addTo(int i, byte[] b, int off, int len) { private void clear() { for (int i = 0; i< numAllBlocks; i++) { buffers[i].clear(); + buffers[i].limit(cellSize); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java index 0c545bebcc..e095602c6e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.server.namenode.INodeFile; import org.apache.hadoop.io.erasurecode.ECSchema; +import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.test.GenericTestUtils; @@ -60,7 +61,7 @@ public class TestErasureCodingPolicies { private Configuration conf; private MiniDFSCluster cluster; private DistributedFileSystem fs; - private static final int BLOCK_SIZE = 1024; + private static final int BLOCK_SIZE = 16 * 1024; private ErasureCodingPolicy ecPolicy; private FSNamesystem namesystem; @@ -78,7 +79,8 @@ public void setupCluster() throws IOException { conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE); DFSTestUtil.enableAllECPolicies(conf); cluster = new MiniDFSCluster.Builder(conf). - numDataNodes(1).build(); + numDataNodes(ecPolicy.getNumDataUnits() + ecPolicy.getNumParityUnits()). + build(); cluster.waitActive(); fs = cluster.getFileSystem(); namesystem = cluster.getNamesystem(); @@ -836,4 +838,48 @@ public void testReplicationPolicy() throws Exception { ecPolicy, fs.getErasureCodingPolicy(subReplicaFile)); fs.delete(subReplicaFile, false); } + + @Test + public void testDifferentErasureCodingPolicyCellSize() throws Exception { + // add policy with cell size 8K + ErasureCodingPolicy newPolicy1 = + new ErasureCodingPolicy(ErasureCodeConstants.RS_3_2_SCHEMA, 8 * 1024); + ErasureCodingPolicy[] policyArray = + new ErasureCodingPolicy[] {newPolicy1}; + AddECPolicyResponse[] responses = fs.addErasureCodingPolicies(policyArray); + assertEquals(1, responses.length); + assertTrue(responses[0].isSucceed()); + newPolicy1 = responses[0].getPolicy(); + + // add policy with cell size 4K + ErasureCodingPolicy newPolicy2 = + new ErasureCodingPolicy(ErasureCodeConstants.RS_3_2_SCHEMA, 4 * 1024); + policyArray = new ErasureCodingPolicy[] {newPolicy2}; + responses = fs.addErasureCodingPolicies(policyArray); + assertEquals(1, responses.length); + assertTrue(responses[0].isSucceed()); + newPolicy2 = responses[0].getPolicy(); + + // enable policies + fs.enableErasureCodingPolicy(newPolicy1.getName()); + fs.enableErasureCodingPolicy(newPolicy2.getName()); + + final Path stripedDir1 = new Path("/striped1"); + final Path stripedDir2 = new Path("/striped2"); + final Path file1 = new Path(stripedDir1, "file"); + final Path file2 = new Path(stripedDir2, "file"); + + fs.mkdirs(stripedDir1); + fs.setErasureCodingPolicy(stripedDir1, newPolicy1.getName()); + fs.mkdirs(stripedDir2); + fs.setErasureCodingPolicy(stripedDir2, newPolicy2.getName()); + + final int fileLength = BLOCK_SIZE * newPolicy1.getNumDataUnits(); + final byte[] bytes = StripedFileTestUtil.generateBytes(fileLength); + DFSTestUtil.writeFile(fs, file1, bytes); + DFSTestUtil.writeFile(fs, file2, bytes); + + fs.delete(stripedDir1, true); + fs.delete(stripedDir2, true); + } }