From a18424e724982d01e0191ca7db713da1d43d1e4d Mon Sep 17 00:00:00 2001 From: Chris Douglas Date: Wed, 3 Sep 2014 14:32:35 -0700 Subject: [PATCH] MAPREDUCE-6063. Correct spill size calculation for spills wrapping the circular buffer. Contributed by zhihai xu. --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../src/main/java/org/apache/hadoop/mapred/MapTask.java | 4 +--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 63bc08be96..0ee07310b5 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -267,6 +267,9 @@ Release 2.6.0 - UNRELEASED MAPREDUCE-5931. Validate SleepJob command line parameters (Gera Shegalov via jlowe) + MAPREDUCE-6063. Correct spill size calculation for spills wrapping the + circular buffer. (zhihai xu via cdouglas) + Release 2.5.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java index dfcbe09383..75b414196d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java @@ -1575,9 +1575,7 @@ private void sortAndSpill() throws IOException, ClassNotFoundException, InterruptedException { //approximate the length of the output file to be the length of the //buffer + header lengths for the partitions - final long size = (bufend >= bufstart - ? bufend - bufstart - : (bufvoid - bufend) + bufstart) + + final long size = distanceTo(bufstart, bufend, bufvoid) + partitions * APPROX_HEADER_LENGTH; FSDataOutputStream out = null; try {