big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-8744. Erasure Coding: the number of chunks in packet is not updated when writing parity data. Contributed by Li Bo

Browse Source
This commit is contained in:
boli2 2015-07-13 00:41:36 -04:00
parent e692c7dd92
commit f4098dfae4
4 changed files with 10 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
View File

@ -341,3 +341,6 @@
HDFS-8484. Erasure coding: Two contiguous blocks occupy IDs belong to same HDFS-8484. Erasure coding: Two contiguous blocks occupy IDs belong to same
striped group. (Walter Su via jing9) striped group. (Walter Su via jing9)
HDFS-8744. Erasure Coding: the number of chunks in packet is not updated
when writing parity data. (Li Bo)

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
View File

@ -419,7 +419,7 @@ protected synchronized void writeChunk(byte[] b, int offset, int len,
currentPacket.writeChecksum(checksum, ckoff, cklen); currentPacket.writeChecksum(checksum, ckoff, cklen);
currentPacket.writeData(b, offset, len); currentPacket.writeData(b, offset, len);
currentPacket.incNumChunks(); currentPacket.incNumChunks(1);
streamer.incBytesCurBlock(len); streamer.incBytesCurBlock(len);
// If packet is full, enqueue it for transmission // If packet is full, enqueue it for transmission

6
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java
View File

@ -259,10 +259,10 @@ synchronized int getNumChunks() {
} }
/** /**
* increase the number of chunks by one * increase the number of chunks by n
*/ */
synchronized void incNumChunks() { synchronized void incNumChunks(int n) {
numChunks++; numChunks += n;
} }
/** /**

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
View File

@ -389,11 +389,13 @@ private List<DFSPacket> generatePackets(
int maxBytesToPacket = p.getMaxChunks() * bytesPerChecksum; int maxBytesToPacket = p.getMaxChunks() * bytesPerChecksum;
int toWrite = byteBuffer.remaining() > maxBytesToPacket ? int toWrite = byteBuffer.remaining() > maxBytesToPacket ?
maxBytesToPacket: byteBuffer.remaining(); maxBytesToPacket: byteBuffer.remaining();
int ckLen = ((toWrite - 1) / bytesPerChecksum + 1) * getChecksumSize(); int chunks = (toWrite - 1) / bytesPerChecksum + 1;
int ckLen = chunks * getChecksumSize();
p.writeChecksum(checksumBuf, ckOff, ckLen); p.writeChecksum(checksumBuf, ckOff, ckLen);
ckOff += ckLen; ckOff += ckLen;
p.writeData(byteBuffer, toWrite); p.writeData(byteBuffer, toWrite);
getCurrentStreamer().incBytesCurBlock(toWrite); getCurrentStreamer().incBytesCurBlock(toWrite);
p.incNumChunks(chunks);
packets.add(p); packets.add(p);
} }
return packets; return packets;