big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-5613. DefaultSpeculator holds and checks hashmap that is alway empty (Gera Shegalov via Sandy Ryza)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1541433 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Sanford Ryza 2013-11-13 09:23:18 +00:00
parent 39def49661
commit e94ed91114
2 changed files with 3 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -167,6 +167,9 @@ Release 2.3.0 - UNRELEASED
MAPREDUCE-1176. FixedLengthInputFormat and FixedLengthRecordReader MAPREDUCE-1176. FixedLengthInputFormat and FixedLengthRecordReader
(Mariappan Asokan and BitsOfInfo via Sandy Ryza) (Mariappan Asokan and BitsOfInfo via Sandy Ryza)
MAPREDUCE-5613. DefaultSpeculator holds and checks hashmap that is always
empty (Gera Shegalov via Sandy Ryza)
OPTIMIZATIONS OPTIMIZATIONS
MAPREDUCE-5484. YarnChild unnecessarily loads job conf twice (Sandy Ryza) MAPREDUCE-5484. YarnChild unnecessarily loads job conf twice (Sandy Ryza)

11
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java
View File

@ -28,7 +28,6 @@
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
@ -75,9 +74,6 @@ public class DefaultSpeculator extends AbstractService implements
private final ConcurrentMap<TaskId, Boolean> runningTasks private final ConcurrentMap<TaskId, Boolean> runningTasks
= new ConcurrentHashMap<TaskId, Boolean>(); = new ConcurrentHashMap<TaskId, Boolean>();
private final Map<Task, AtomicBoolean> pendingSpeculations
= new ConcurrentHashMap<Task, AtomicBoolean>();
// Used to track any TaskAttempts that aren't heart-beating for a while, so // Used to track any TaskAttempts that aren't heart-beating for a while, so
// that we can aggressively speculate instead of waiting for task-timeout. // that we can aggressively speculate instead of waiting for task-timeout.
private final ConcurrentMap<TaskAttemptId, TaskAttemptHistoryStatistics> private final ConcurrentMap<TaskAttemptId, TaskAttemptHistoryStatistics>
@ -328,13 +324,6 @@ protected void statusUpdate(TaskAttemptStatus reportedStatus, long timestamp) {
estimator.updateAttempt(reportedStatus, timestamp); estimator.updateAttempt(reportedStatus, timestamp);
// If the task is already known to be speculation-bait, don't do anything
if (pendingSpeculations.get(task) != null) {
if (pendingSpeculations.get(task).get()) {
return;
}
}
if (stateString.equals(TaskAttemptState.RUNNING.name())) { if (stateString.equals(TaskAttemptState.RUNNING.name())) {
runningTasks.putIfAbsent(taskID, Boolean.TRUE); runningTasks.putIfAbsent(taskID, Boolean.TRUE);
} else { } else {