From d99c2958dd33fcbebee9208ab8388faa0725c699 Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Wed, 4 Apr 2012 14:17:22 +0000 Subject: [PATCH] MAPREDUCE-3672. Killed maps shouldn't be counted towards JobCounter.NUM_FAILED_MAPS. (Anupam Seth via tgraves) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1309401 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../v2/app/job/impl/TaskAttemptImpl.java | 26 +++++++++++++++++-- .../apache/hadoop/mapreduce/JobCounter.java | 2 ++ .../hadoop/mapreduce/JobCounter.properties | 2 ++ .../apache/hadoop/mapred/TestJobCleanup.java | 5 ++++ .../v2/TestSpeculativeExecution.java | 4 ++- 6 files changed, 39 insertions(+), 3 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 6241943baf..38f935f47d 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -240,6 +240,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4020. Web services returns incorrect JSON for deep queue tree (Anupam Seth via tgraves) + MAPREDUCE-3672. Killed maps shouldn't be counted towards + JobCounter.NUM_FAILED_MAPS. (Anupam Seth via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java index 33dbb13523..53c7211d62 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java @@ -988,6 +988,23 @@ public abstract class TaskAttemptImpl implements } return jce; } + + private static JobCounterUpdateEvent createJobCounterUpdateEventTAKilled( + TaskAttemptImpl taskAttempt) { + TaskType taskType = taskAttempt.getID().getTaskId().getTaskType(); + JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskAttempt.getID().getTaskId().getJobId()); + + long slotMillisIncrement = computeSlotMillis(taskAttempt); + + if (taskType == TaskType.MAP) { + jce.addCounterUpdate(JobCounter.NUM_KILLED_MAPS, 1); + jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_MAPS, slotMillisIncrement); + } else { + jce.addCounterUpdate(JobCounter.NUM_KILLED_REDUCES, 1); + jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_REDUCES, slotMillisIncrement); + } + return jce; + } private static TaskAttemptUnsuccessfulCompletionEvent @@ -1214,8 +1231,13 @@ public abstract class TaskAttemptImpl implements TaskAttemptUnsuccessfulCompletionEvent tauce = createTaskAttemptUnsuccessfulCompletionEvent(taskAttempt, finalState); - taskAttempt.eventHandler + if(finalState == TaskAttemptState.FAILED) { + taskAttempt.eventHandler .handle(createJobCounterUpdateEventTAFailed(taskAttempt)); + } else if(finalState == TaskAttemptState.KILLED) { + taskAttempt.eventHandler + .handle(createJobCounterUpdateEventTAKilled(taskAttempt)); + } taskAttempt.eventHandler.handle(new JobHistoryEvent( taskAttempt.attemptId.getTaskId().getJobId(), tauce)); } else { @@ -1441,7 +1463,7 @@ public abstract class TaskAttemptImpl implements taskAttempt.setFinishTime(); if (taskAttempt.getLaunchTime() != 0) { taskAttempt.eventHandler - .handle(createJobCounterUpdateEventTAFailed(taskAttempt)); + .handle(createJobCounterUpdateEventTAKilled(taskAttempt)); TaskAttemptUnsuccessfulCompletionEvent tauce = createTaskAttemptUnsuccessfulCompletionEvent(taskAttempt, TaskAttemptState.KILLED); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java index 784e1d5401..47c2872d3b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java @@ -27,6 +27,8 @@ import org.apache.hadoop.classification.InterfaceStability; public enum JobCounter { NUM_FAILED_MAPS, NUM_FAILED_REDUCES, + NUM_KILLED_MAPS, + NUM_KILLED_REDUCES, TOTAL_LAUNCHED_MAPS, TOTAL_LAUNCHED_REDUCES, OTHER_LOCAL_MAPS, diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties index 98e9279958..59dff06057 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties @@ -16,6 +16,8 @@ CounterGroupName= Job Counters NUM_FAILED_MAPS.name= Failed map tasks NUM_FAILED_REDUCES.name= Failed reduce tasks +NUM_KILLED_MAPS.name= Killed map tasks +NUM_KILLED_REDUCES.name= Killed reduce tasks TOTAL_LAUNCHED_MAPS.name= Launched map tasks TOTAL_LAUNCHED_REDUCES.name= Launched reduce tasks OTHER_LOCAL_MAPS.name= Other local map tasks diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCleanup.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCleanup.java index 6139fdb05f..c715956523 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCleanup.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCleanup.java @@ -195,6 +195,8 @@ public class TestJobCleanup { RunningJob job = jobClient.submitJob(jc); JobID id = job.getID(); job.waitForCompletion(); + Counters counters = job.getCounters(); + assertTrue("No. of failed maps should be 1",counters.getCounter(JobCounter.NUM_FAILED_MAPS) == 1); if (fileName != null) { Path testFile = new Path(outDir, fileName); @@ -240,6 +242,9 @@ public class TestJobCleanup { job.killJob(); // kill the job job.waitForCompletion(); // wait for the job to complete + + counters = job.getCounters(); + assertTrue("No. of killed maps should be 1", counters.getCounter(JobCounter.NUM_KILLED_MAPS) == 1); if (fileName != null) { Path testFile = new Path(outDir, fileName); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java index 12bb5ac0e7..90f7cafc63 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java @@ -234,8 +234,10 @@ public class TestSpeculativeExecution { .getValue()); Assert.assertEquals(2, counters.findCounter(JobCounter.TOTAL_LAUNCHED_REDUCES) .getValue()); - Assert.assertEquals(1, counters.findCounter(JobCounter.NUM_FAILED_MAPS) + Assert.assertEquals(0, counters.findCounter(JobCounter.NUM_FAILED_MAPS) .getValue()); + Assert.assertEquals(1, counters.findCounter(JobCounter.NUM_KILLED_MAPS) + .getValue()); /*---------------------------------------------------------------------- * Test that Reducer speculates if REDUCE_SPECULATIVE is true and