From ac7d0604bc73c0925eff240ad9837e14719d57b7 Mon Sep 17 00:00:00 2001 From: Ravi Prakash Date: Wed, 9 Aug 2017 15:39:52 -0700 Subject: [PATCH] MAPREDUCE-6923. Optimize MapReduce Shuffle I/O for small partitions. Contributed by Robert Schmidtke. --- .../java/org/apache/hadoop/mapred/FadvisedFileRegion.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java index cb9b5e088e..79045f9555 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java @@ -111,7 +111,10 @@ long customShuffleTransfer(WritableByteChannel target, long position) long trans = actualCount; int readSize; - ByteBuffer byteBuffer = ByteBuffer.allocate(this.shuffleBufferSize); + ByteBuffer byteBuffer = ByteBuffer.allocate( + Math.min( + this.shuffleBufferSize, + trans > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) trans)); while(trans > 0L && (readSize = fileChannel.read(byteBuffer, this.position+position)) > 0) {