From 8c1adeaa26a7eaaca891dccdb71ff085e598a7de Mon Sep 17 00:00:00 2001 From: Junping Du Date: Mon, 11 Jan 2016 06:12:19 -0800 Subject: [PATCH] MAPREDUCE-6068. Illegal progress value warnings in map tasks. Contributed by Binglin Chang. --- hadoop-mapreduce-project/CHANGES.txt | 7 +++++-- .../src/main/java/org/apache/hadoop/mapred/Merger.java | 6 +++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 5422646823..1c9bd4a371 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -677,8 +677,8 @@ Release 2.8.0 - UNRELEASED MAPREDUCE-6550. archive-logs tool changes log ownership to the Yarn user when using DefaultContainerExecutor (rkanter) - MAPREDUCE-6419. JobHistoryServer doesn't sort properly based on Job ID when Job id's - exceed 9999. (Mohammad Shahid Khan via rohithsharmaks) + MAPREDUCE-6419. JobHistoryServer doesn't sort properly based on Job ID when Job id's + exceed 9999. (Mohammad Shahid Khan via rohithsharmaks) MAPREDUCE-6589. TestTaskLog outputs a log under directory other than target/test-dir. (aajisaka) @@ -686,6 +686,9 @@ Release 2.8.0 - UNRELEASED MAPREDUCE-6593. TestJobHistoryEventHandler.testTimelineEventHandling fails on trunk because of NPE. (Naganarasimha G R via aajisaka) + MAPREDUCE-6068. Illegal progress value warnings in map tasks. (Binglin Chang + via junping_du) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Merger.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Merger.java index fffa92a021..3667e3ca88 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Merger.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Merger.java @@ -519,7 +519,7 @@ private void adjustPriorityQueue(Segment reader) throws IOException{ boolean hasNext = reader.nextRawKey(); long endPos = reader.getReader().bytesRead; totalBytesProcessed += endPos - startPos; - mergeProgress.set(totalBytesProcessed * progPerByte); + mergeProgress.set(Math.min(1.0f, totalBytesProcessed * progPerByte)); if (hasNext) { adjustTop(); } else { @@ -571,7 +571,7 @@ public boolean next() throws IOException { } long endPos = minSegment.getReader().bytesRead; totalBytesProcessed += endPos - startPos; - mergeProgress.set(totalBytesProcessed * progPerByte); + mergeProgress.set(Math.min(1.0f, totalBytesProcessed * progPerByte)); return true; } @@ -698,7 +698,7 @@ RawKeyValueIterator merge(Class keyClass, Class valueClass, totalBytesProcessed += startBytes; if (totalBytes != 0) - mergeProgress.set(totalBytesProcessed * progPerByte); + mergeProgress.set(Math.min(1.0f, totalBytesProcessed * progPerByte)); else mergeProgress.set(1.0f); // Last pass and no segments left - we're done