big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-8813. Erasure Coding: Client no need to decode missing parity blocks. Contributed by Walter Su.

Browse Source
This commit is contained in:
Jing Zhao 2015-07-23 11:42:02 -07:00
parent 5956d23b64
commit c2c26e6ea7
3 changed files with 10 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
View File

@ -370,3 +370,6 @@
HDFS-8781. Erasure Coding: Correctly handle BlockManager#InvalidateBlocks for HDFS-8781. Erasure Coding: Correctly handle BlockManager#InvalidateBlocks for
striped block. (Yi Liu via jing9) striped block. (Yi Liu via jing9)
HDFS-8813. Erasure Coding: Client no need to decode missing parity blocks.
(Walter Su via jing9)

7
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
View File

@ -867,8 +867,13 @@ void decode() {
for (int i = 0; i < alignedStripe.chunks.length; i++) { for (int i = 0; i < alignedStripe.chunks.length; i++) {
if (alignedStripe.chunks[i] != null && if (alignedStripe.chunks[i] != null &&
alignedStripe.chunks[i].state == StripingChunk.MISSING) { alignedStripe.chunks[i].state == StripingChunk.MISSING) {
decodeIndices[pos++] = StripedBlockUtil.convertIndex4Decode(i, int decodeIndex = StripedBlockUtil.convertIndex4Decode(i,
dataBlkNum, parityBlkNum); dataBlkNum, parityBlkNum);
if (i < dataBlkNum) {
decodeIndices[pos++] = decodeIndex;
} else {
decodeInputs[decodeIndex] = null;
}
} }
} }
decodeIndices = Arrays.copyOf(decodeIndices, pos); decodeIndices = Arrays.copyOf(decodeIndices, pos);

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
View File

@ -326,7 +326,7 @@ public static void decodeAndFillBuffer(final byte[][] decodeInputs,
// Step 1: prepare indices and output buffers for missing data units // Step 1: prepare indices and output buffers for missing data units
int[] decodeIndices = new int[parityBlkNum]; int[] decodeIndices = new int[parityBlkNum];
int pos = 0; int pos = 0;
for (int i = 0; i < alignedStripe.chunks.length; i++) { for (int i = 0; i < dataBlkNum; i++) {
if (alignedStripe.chunks[i] != null && if (alignedStripe.chunks[i] != null &&
alignedStripe.chunks[i].state == StripingChunk.MISSING){ alignedStripe.chunks[i].state == StripingChunk.MISSING){
decodeIndices[pos++] = convertIndex4Decode(i, dataBlkNum, parityBlkNum); decodeIndices[pos++] = convertIndex4Decode(i, dataBlkNum, parityBlkNum);