big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-5821. Avoid unintentional reallocation of byte arrays in segments

Browse Source 
during merge. Contributed by Todd Lipcon


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1594654 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Christopher Douglas 2014-05-14 17:51:11 +00:00
parent 561c014e77
commit 5bf7ef839b
2 changed files with 8 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -273,6 +273,9 @@ Release 2.4.1 - UNRELEASED
MAPREDUCE-5835. Killing Task might cause the job to go to ERROR state MAPREDUCE-5835. Killing Task might cause the job to go to ERROR state
(Ming Ma via jlowe) (Ming Ma via jlowe)
MAPREDUCE-5821. Avoid unintentional reallocation of byte arrays in segments
during merge. (Todd Lipcon via cdouglas)
Release 2.4.0 - 2014-04-07 Release 2.4.0 - 2014-04-07
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

8
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Merger.java
View File

@ -537,6 +537,8 @@ public boolean next() throws IOException {
} }
} }
minSegment = top(); minSegment = top();
long startPos = minSegment.getPosition();
key = minSegment.getKey();
if (!minSegment.inMemory()) { if (!minSegment.inMemory()) {
//When we load the value from an inmemory segment, we reset //When we load the value from an inmemory segment, we reset
//the "value" DIB in this class to the inmem segment's byte[]. //the "value" DIB in this class to the inmem segment's byte[].
@ -547,11 +549,11 @@ public boolean next() throws IOException {
//segment, we reset the "value" DIB to the byte[] in that (so //segment, we reset the "value" DIB to the byte[] in that (so
//we reuse the disk segment DIB whenever we consider //we reuse the disk segment DIB whenever we consider
//a disk segment). //a disk segment).
minSegment.getValue(diskIFileValue);
value.reset(diskIFileValue.getData(), diskIFileValue.getLength()); value.reset(diskIFileValue.getData(), diskIFileValue.getLength());
} else {
minSegment.getValue(value);
} }
long startPos = minSegment.getPosition();
key = minSegment.getKey();
minSegment.getValue(value);
long endPos = minSegment.getPosition(); long endPos = minSegment.getPosition();
totalBytesProcessed += endPos - startPos; totalBytesProcessed += endPos - startPos;
mergeProgress.set(totalBytesProcessed * progPerByte); mergeProgress.set(totalBytesProcessed * progPerByte);