big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-12392. Writing striped file failed due to different cell size. Contributed by Sammi Chen

Browse Source
This commit is contained in:
Kai Zheng 2017-09-06 13:29:52 +08:00
parent d4035d42f0
commit d7f27043ce
2 changed files with 50 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
View File

@ -215,6 +215,7 @@ class CellBuffers {
buffers = new ByteBuffer[numAllBlocks];
for (int i = 0; i < buffers.length; i++) {
buffers[i] = BUFFER_POOL.getBuffer(useDirectBuffer(), cellSize);
buffers[i].limit(cellSize);
}
}
@ -237,6 +238,7 @@ private int addTo(int i, byte[] b, int off, int len) {
private void clear() {
for (int i = 0; i< numAllBlocks; i++) {
buffers[i].clear();
buffers[i].limit(cellSize);
}
}

50
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
View File

@ -35,6 +35,7 @@
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
@ -60,7 +61,7 @@ public class TestErasureCodingPolicies {
private Configuration conf;
private MiniDFSCluster cluster;
private DistributedFileSystem fs;
private static final int BLOCK_SIZE = 1024;
private static final int BLOCK_SIZE = 16 * 1024;
private ErasureCodingPolicy ecPolicy;
private FSNamesystem namesystem;
@ -78,7 +79,8 @@ public void setupCluster() throws IOException {
conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
DFSTestUtil.enableAllECPolicies(conf);
cluster = new MiniDFSCluster.Builder(conf).
numDataNodes(1).build();
numDataNodes(ecPolicy.getNumDataUnits() + ecPolicy.getNumParityUnits()).
build();
cluster.waitActive();
fs = cluster.getFileSystem();
namesystem = cluster.getNamesystem();
@ -836,4 +838,48 @@ public void testReplicationPolicy() throws Exception {
ecPolicy, fs.getErasureCodingPolicy(subReplicaFile));
fs.delete(subReplicaFile, false);
}
@Test
public void testDifferentErasureCodingPolicyCellSize() throws Exception {
// add policy with cell size 8K
ErasureCodingPolicy newPolicy1 =
new ErasureCodingPolicy(ErasureCodeConstants.RS_3_2_SCHEMA, 8 * 1024);
ErasureCodingPolicy[] policyArray =
new ErasureCodingPolicy[] {newPolicy1};
AddECPolicyResponse[] responses = fs.addErasureCodingPolicies(policyArray);
assertEquals(1, responses.length);
assertTrue(responses[0].isSucceed());
newPolicy1 = responses[0].getPolicy();
// add policy with cell size 4K
ErasureCodingPolicy newPolicy2 =
new ErasureCodingPolicy(ErasureCodeConstants.RS_3_2_SCHEMA, 4 * 1024);
policyArray = new ErasureCodingPolicy[] {newPolicy2};
responses = fs.addErasureCodingPolicies(policyArray);
assertEquals(1, responses.length);
assertTrue(responses[0].isSucceed());
newPolicy2 = responses[0].getPolicy();
// enable policies
fs.enableErasureCodingPolicy(newPolicy1.getName());
fs.enableErasureCodingPolicy(newPolicy2.getName());
final Path stripedDir1 = new Path("/striped1");
final Path stripedDir2 = new Path("/striped2");
final Path file1 = new Path(stripedDir1, "file");
final Path file2 = new Path(stripedDir2, "file");
fs.mkdirs(stripedDir1);
fs.setErasureCodingPolicy(stripedDir1, newPolicy1.getName());
fs.mkdirs(stripedDir2);
fs.setErasureCodingPolicy(stripedDir2, newPolicy2.getName());
final int fileLength = BLOCK_SIZE * newPolicy1.getNumDataUnits();
final byte[] bytes = StripedFileTestUtil.generateBytes(fileLength);
DFSTestUtil.writeFile(fs, file1, bytes);
DFSTestUtil.writeFile(fs, file2, bytes);
fs.delete(stripedDir1, true);
fs.delete(stripedDir2, true);
}
}