big-data/hadoop
big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Projects Wiki Activity

HDFS-8804. Erasure Coding: use DirectBufferPool in DFSStripedInputStream for buffer allocation. Contributed by Jing Zhao.

Browse Source
This commit is contained in:
Jing Zhao 2015-08-03 17:03:15 -07:00
parent ba90c02853
commit 9312b168e2
2 changed files with 33 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
View File

@ -382,3 +382,6 @@
HDFS-8202. Improve end to end stirpping file test to add erasure recovering HDFS-8202. Improve end to end stirpping file test to add erasure recovering
test. (Xinwei Qin via zhz) test. (Xinwei Qin via zhz)
HDFS-8804. Erasure Coding: use DirectBufferPool in DFSStripedInputStream for
buffer allocation. (jing9)

33
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
View File

@ -39,6 +39,7 @@ import org.apache.hadoop.io.erasurecode.CodecUtil;
import org.apache.hadoop.io.erasurecode.ECSchema; import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
import org.apache.hadoop.util.DirectBufferPool;
import java.io.EOFException; import java.io.EOFException;
import java.io.IOException; import java.io.IOException;
@ -136,6 +137,8 @@ public class DFSStripedInputStream extends DFSInputStream {
} }
} }
private static final DirectBufferPool bufferPool = new DirectBufferPool();
private final BlockReaderInfo[] blockReaders; private final BlockReaderInfo[] blockReaders;
private final int cellSize; private final int cellSize;
private final short dataBlkNum; private final short dataBlkNum;
@ -143,6 +146,7 @@ public class DFSStripedInputStream extends DFSInputStream {
private final int groupSize; private final int groupSize;
/** the buffer for a complete stripe */ /** the buffer for a complete stripe */
private ByteBuffer curStripeBuf; private ByteBuffer curStripeBuf;
private ByteBuffer parityBuf;
private final ECSchema schema; private final ECSchema schema;
private final RawErasureDecoder decoder; private final RawErasureDecoder decoder;
@ -177,12 +181,20 @@ public class DFSStripedInputStream extends DFSInputStream {
private void resetCurStripeBuffer() { private void resetCurStripeBuffer() {
if (curStripeBuf == null) { if (curStripeBuf == null) {
curStripeBuf = ByteBuffer.allocateDirect(cellSize * dataBlkNum); curStripeBuf = bufferPool.getBuffer(cellSize * dataBlkNum);
} }
curStripeBuf.clear(); curStripeBuf.clear();
curStripeRange = new StripeRange(0, 0); curStripeRange = new StripeRange(0, 0);
} }
private ByteBuffer getParityBuffer() {
if (parityBuf == null) {
parityBuf = bufferPool.getBuffer(cellSize * parityBlkNum);
}
parityBuf.clear();
return parityBuf;
}
/** /**
* When seeking into a new block group, create blockReader for each internal * When seeking into a new block group, create blockReader for each internal
* block in the group. * block in the group.
@ -204,6 +216,19 @@ public class DFSStripedInputStream extends DFSInputStream {
currentLocatedBlock = targetBlockGroup; currentLocatedBlock = targetBlockGroup;
} }
@Override
public synchronized void close() throws IOException {
super.close();
if (curStripeBuf != null) {
bufferPool.returnBuffer(curStripeBuf);
curStripeBuf = null;
}
if (parityBuf != null) {
bufferPool.returnBuffer(parityBuf);
parityBuf = null;
}
}
/** /**
* Extend the super method with the logic of switching between cells. * Extend the super method with the logic of switching between cells.
* When reaching the end of a cell, proceed to the next cell and read it * When reaching the end of a cell, proceed to the next cell and read it
@ -830,8 +855,10 @@ public class DFSStripedInputStream extends DFSInputStream {
} }
final int decodeIndex = StripedBlockUtil.convertIndex4Decode(index, final int decodeIndex = StripedBlockUtil.convertIndex4Decode(index,
dataBlkNum, parityBlkNum); dataBlkNum, parityBlkNum);
decodeInputs[decodeIndex] = ByteBuffer.allocateDirect( ByteBuffer buf = getParityBuffer().duplicate();
(int) alignedStripe.range.spanInBlock); buf.position(cellSize * decodeIndex);
buf.limit(cellSize * decodeIndex + (int) alignedStripe.range.spanInBlock);
decodeInputs[decodeIndex] = buf.slice();
alignedStripe.chunks[index] = new StripingChunk(decodeInputs[decodeIndex]); alignedStripe.chunks[index] = new StripingChunk(decodeInputs[decodeIndex]);
return true; return true;
} }