diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index cb16faec11..f3f29ae2b5 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -238,6 +238,9 @@ Release 2.0.5-beta - UNRELEASED MAPREDUCE-4716. TestHsWebServicesJobsQuery.testJobsQueryStateInvalid fails with jdk7. (tgraves via tucu) + MAPREDUCE-5028. Maps fail when io.sort.mb is set to high value. + (kkambatl via tucu) + Release 2.0.4-alpha - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java index cd4ba51c31..496ca2e656 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java @@ -1165,8 +1165,9 @@ private void setEquator(int pos) { equator = pos; // set index prior to first entry, aligned at meta boundary final int aligned = pos - (pos % METASIZE); - kvindex = - ((aligned - METASIZE + kvbuffer.length) % kvbuffer.length) / 4; + // Cast one of the operands to long to ensure large values don't cause int + // overflow + kvindex = (int) (((long) aligned - METASIZE + kvbuffer.length) % kvbuffer.length) / 4; if (LOG.isInfoEnabled()) { LOG.info("(EQUATOR) " + pos + " kvi " + kvindex + "(" + (kvindex * 4) + ")"); @@ -1183,8 +1184,9 @@ private void resetSpill() { bufstart = bufend = e; final int aligned = e - (e % METASIZE); // set start/end to point to first meta record - kvstart = kvend = - ((aligned - METASIZE + kvbuffer.length) % kvbuffer.length) / 4; + // Cast one of the operands to long to ensure large values don't cause int + // overflow + kvstart = kvend = (int) (((long) aligned - METASIZE + kvbuffer.length) % kvbuffer.length) / 4; if (LOG.isInfoEnabled()) { LOG.info("(RESET) equator " + e + " kv " + kvstart + "(" + (kvstart * 4) + ")" + " kvi " + kvindex + "(" + (kvindex * 4) + ")"); @@ -1748,7 +1750,7 @@ public void reset(byte[] buffer, int start, int length) { this.start = 0; } - super.reset(this.buffer, this.start, this.length); + super.reset(this.buffer, this.start, this.length - this.start); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/ReduceContextImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/ReduceContextImpl.java index 7ad08e956d..8736de52f8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/ReduceContextImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/ReduceContextImpl.java @@ -205,7 +205,8 @@ public VALUEIN next() { if (backupStore.hasNext()) { backupStore.next(); DataInputBuffer next = backupStore.nextValue(); - buffer.reset(next.getData(), next.getPosition(), next.getLength()); + buffer.reset(next.getData(), next.getPosition(), next.getLength() + - next.getPosition()); value = valueDeserializer.deserialize(value); return value; } else { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/InMemoryReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/InMemoryReader.java index 543ff3f9cc..6def88b97b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/InMemoryReader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/InMemoryReader.java @@ -49,14 +49,14 @@ public InMemoryReader(MergeManagerImpl merger, TaskAttemptID taskAttemptId, buffer = data; bufferSize = (int)fileLength; - memDataIn.reset(buffer, start, length); + memDataIn.reset(buffer, start, length - start); this.start = start; this.length = length; } @Override public void reset(int offset) { - memDataIn.reset(buffer, start + offset, length); + memDataIn.reset(buffer, start + offset, length - start - offset); bytesRead = offset; eof = false; }