big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-8460. Erasure Coding: stateful read result doesn't match data occasionally because of flawed test. Contributed by Walter Su.

Browse Source
This commit is contained in:
Zhe Zhang 2015-06-05 13:39:37 -07:00
parent c0929ab3c2
commit 2eee19cd1d
3 changed files with 10 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
View File

@ -286,3 +286,6 @@
HDFS-8319. Erasure Coding: support decoding for stateful read. HDFS-8319. Erasure Coding: support decoding for stateful read.
(Jing Zhao via zhz) (Jing Zhao via zhz)
HDFS-8460. Erasure Coding: stateful read result doesn't match data
occasionally because of flawed test. (Walter Su via zhz)

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
View File

@ -1859,6 +1859,7 @@ public static StorageReceivedDeletedBlocks[] makeReportForReceivedBlock(
/** /**
* Creates the metadata of a file in striped layout. This method only * Creates the metadata of a file in striped layout. This method only
* manipulates the NameNode state without injecting data to DataNode. * manipulates the NameNode state without injecting data to DataNode.
* You should disable periodical heartbeat before use this.
* @param file Path of the file to create * @param file Path of the file to create
* @param dir Parent path of the file * @param dir Parent path of the file
* @param numBlocks Number of striped block groups to add to the file * @param numBlocks Number of striped block groups to add to the file
@ -1908,6 +1909,7 @@ public static void createStripedFile(MiniDFSCluster cluster, Path file, Path dir
* Adds a striped block group to a file. This method only manipulates NameNode * Adds a striped block group to a file. This method only manipulates NameNode
* states of the file and the block without injecting data to DataNode. * states of the file and the block without injecting data to DataNode.
* It does mimic block reports. * It does mimic block reports.
* You should disable periodical heartbeat before use this.
* @param dataNodes List DataNodes to host the striped block group * @param dataNodes List DataNodes to host the striped block group
* @param previous Previous block in the file * @param previous Previous block in the file
* @param numStripes Number of stripes in each block group * @param numStripes Number of stripes in each block group

5
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
View File

@ -32,6 +32,8 @@
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset; import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
import org.apache.hadoop.hdfs.server.namenode.ErasureCodingSchemaManager; import org.apache.hadoop.hdfs.server.namenode.ErasureCodingSchemaManager;
import org.apache.hadoop.hdfs.util.StripedBlockUtil; import org.apache.hadoop.hdfs.util.StripedBlockUtil;
@ -71,6 +73,9 @@ public void setup() throws IOException {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes( cluster = new MiniDFSCluster.Builder(conf).numDataNodes(
DATA_BLK_NUM + PARITY_BLK_NUM).build(); DATA_BLK_NUM + PARITY_BLK_NUM).build();
cluster.waitActive(); cluster.waitActive();
for (DataNode dn : cluster.getDataNodes()) {
DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
}
fs = cluster.getFileSystem(); fs = cluster.getFileSystem();
fs.mkdirs(dirPath); fs.mkdirs(dirPath);
fs.getClient().createErasureCodingZone(dirPath.toString(), null, CELLSIZE); fs.getClient().createErasureCodingZone(dirPath.toString(), null, CELLSIZE);