big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-5028. Maps fail when io.sort.mb is set to high value. (kkambatl via tucu)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1457918 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Alejandro Abdelnur 2013-03-18 18:56:40 +00:00
parent 5ac6abe107
commit 53e7aaa6dd
4 changed files with 14 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -238,6 +238,9 @@ Release 2.0.5-beta - UNRELEASED
MAPREDUCE-4716. TestHsWebServicesJobsQuery.testJobsQueryStateInvalid MAPREDUCE-4716. TestHsWebServicesJobsQuery.testJobsQueryStateInvalid
fails with jdk7. (tgraves via tucu) fails with jdk7. (tgraves via tucu)
MAPREDUCE-5028. Maps fail when io.sort.mb is set to high value.
(kkambatl via tucu)
Release 2.0.4-alpha - UNRELEASED Release 2.0.4-alpha - UNRELEASED
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

12
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
View File

@ -1165,8 +1165,9 @@ private void setEquator(int pos) {
equator = pos; equator = pos;
// set index prior to first entry, aligned at meta boundary // set index prior to first entry, aligned at meta boundary
final int aligned = pos - (pos % METASIZE); final int aligned = pos - (pos % METASIZE);
kvindex = // Cast one of the operands to long to ensure large values don't cause int
((aligned - METASIZE + kvbuffer.length) % kvbuffer.length) / 4; // overflow
kvindex = (int) (((long) aligned - METASIZE + kvbuffer.length) % kvbuffer.length) / 4;
if (LOG.isInfoEnabled()) { if (LOG.isInfoEnabled()) {
LOG.info("(EQUATOR) " + pos + " kvi " + kvindex + LOG.info("(EQUATOR) " + pos + " kvi " + kvindex +
"(" + (kvindex * 4) + ")"); "(" + (kvindex * 4) + ")");
@ -1183,8 +1184,9 @@ private void resetSpill() {
bufstart = bufend = e; bufstart = bufend = e;
final int aligned = e - (e % METASIZE); final int aligned = e - (e % METASIZE);
// set start/end to point to first meta record // set start/end to point to first meta record
kvstart = kvend = // Cast one of the operands to long to ensure large values don't cause int
((aligned - METASIZE + kvbuffer.length) % kvbuffer.length) / 4; // overflow
kvstart = kvend = (int) (((long) aligned - METASIZE + kvbuffer.length) % kvbuffer.length) / 4;
if (LOG.isInfoEnabled()) { if (LOG.isInfoEnabled()) {
LOG.info("(RESET) equator " + e + " kv " + kvstart + "(" + LOG.info("(RESET) equator " + e + " kv " + kvstart + "(" +
(kvstart * 4) + ")" + " kvi " + kvindex + "(" + (kvindex * 4) + ")"); (kvstart * 4) + ")" + " kvi " + kvindex + "(" + (kvindex * 4) + ")");
@ -1748,7 +1750,7 @@ public void reset(byte[] buffer, int start, int length) {
this.start = 0; this.start = 0;
} }
super.reset(this.buffer, this.start, this.length); super.reset(this.buffer, this.start, this.length - this.start);
} }
} }

3
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/ReduceContextImpl.java
View File

@ -205,7 +205,8 @@ public VALUEIN next() {
if (backupStore.hasNext()) { if (backupStore.hasNext()) {
backupStore.next(); backupStore.next();
DataInputBuffer next = backupStore.nextValue(); DataInputBuffer next = backupStore.nextValue();
buffer.reset(next.getData(), next.getPosition(), next.getLength()); buffer.reset(next.getData(), next.getPosition(), next.getLength()
- next.getPosition());
value = valueDeserializer.deserialize(value); value = valueDeserializer.deserialize(value);
return value; return value;
} else { } else {

4
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/InMemoryReader.java
View File

@ -49,14 +49,14 @@ public InMemoryReader(MergeManagerImpl<K,V> merger, TaskAttemptID taskAttemptId,
buffer = data; buffer = data;
bufferSize = (int)fileLength; bufferSize = (int)fileLength;
memDataIn.reset(buffer, start, length); memDataIn.reset(buffer, start, length - start);
this.start = start; this.start = start;
this.length = length; this.length = length;
} }
@Override @Override
public void reset(int offset) { public void reset(int offset) {
memDataIn.reset(buffer, start + offset, length); memDataIn.reset(buffer, start + offset, length - start - offset);
bytesRead = offset; bytesRead = offset;
eof = false; eof = false;
} }