diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 08d1e35697..8eb5226a80 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -171,6 +171,9 @@ Release 2.3.0 - UNRELEASED MAPREDUCE-5487. In task processes, JobConf is unnecessarily loaded again in Limits (Sandy Ryza) + MAPREDUCE-5601. ShuffleHandler fadvises file regions as DONTNEED even when + fetch fails (Sandy Ryza) + BUG FIXES MAPREDUCE-5316. job -list-attempt-ids command does not handle illegal diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java index aaf49b5771..20b2bf479d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java @@ -69,6 +69,14 @@ public class FadvisedFileRegion extends DefaultFileRegion { if (readaheadRequest != null) { readaheadRequest.cancel(); } + super.releaseExternalResources(); + } + + /** + * Call when the transfer completes successfully so we can advise the OS that + * we don't need the region to be cached anymore. + */ + public void transferSuccessful() { if (manageOsCache && getCount() > 0) { try { NativeIO.POSIX.posixFadviseIfPossible(identifier, @@ -78,6 +86,5 @@ public class FadvisedFileRegion extends DefaultFileRegion { LOG.warn("Failed to manage OS cache for " + identifier, t); } } - super.releaseExternalResources(); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java index 9f377e23ac..f9f924bd24 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java @@ -625,6 +625,9 @@ public class ShuffleHandler extends AuxiliaryService { // attribute to appropriate spill output @Override public void operationComplete(ChannelFuture future) { + if (future.isSuccess()) { + partition.transferSuccessful(); + } partition.releaseExternalResources(); } });