From 3bbd67173eaf776b7b5e4af8a24dd8738e02fed0 Mon Sep 17 00:00:00 2001 From: Arun Murthy Date: Wed, 12 Mar 2014 22:04:48 +0000 Subject: [PATCH] MAPREDUCE-5553. Allow users to easily access completed/pending/successful/failed tasks on MR AM web-ui. Contributed by Paul Han. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1576946 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 4 ++++ .../mapreduce/v2/app/webapp/AMParams.java | 1 + .../mapreduce/v2/app/webapp/AMWebApp.java | 2 +- .../mapreduce/v2/app/webapp/JobBlock.java | 23 ++++++++----------- .../mapreduce/v2/app/webapp/TasksBlock.java | 20 ++++++++++++++++ .../hadoop/mapreduce/v2/util/MRApps.java | 22 ++++++++++++++++++ .../hadoop/mapreduce/v2/util/TestMRApps.java | 10 ++++++++ 7 files changed, 68 insertions(+), 14 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index f4d903244d..71b7196f40 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -188,6 +188,10 @@ Release 2.4.0 - UNRELEASED MAPREDUCE-5773. Provide dedicated MRAppMaster syslog length limit (Gera Shegalov via jlowe) + MAPREDUCE-5553. Allow users to easily access + completed/pending/successful/failed tasks on MR AM web-ui. (Paul Han via + acmurthy) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMParams.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMParams.java index ad8a15d45a..2ca7ff5471 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMParams.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMParams.java @@ -27,6 +27,7 @@ public interface AMParams { static final String JOB_ID = "job.id"; static final String TASK_ID = "task.id"; static final String TASK_TYPE = "task.type"; + static final String TASK_STATE = "task.state"; static final String ATTEMPT_STATE = "attempt.state"; static final String COUNTER_GROUP = "counter.group"; static final String COUNTER_NAME = "counter.name"; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebApp.java index e3b62c71ec..ee77c8a391 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebApp.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebApp.java @@ -40,7 +40,7 @@ public void setup() { route(pajoin("/jobcounters", JOB_ID), AppController.class, "jobCounters"); route(pajoin("/singlejobcounter",JOB_ID, COUNTER_GROUP, COUNTER_NAME), AppController.class, "singleJobCounter"); - route(pajoin("/tasks", JOB_ID, TASK_TYPE), AppController.class, "tasks"); + route(pajoin("/tasks", JOB_ID, TASK_TYPE, TASK_STATE), AppController.class, "tasks"); route(pajoin("/attempts", JOB_ID, TASK_TYPE, ATTEMPT_STATE), AppController.class, "attempts"); route(pajoin("/task", TASK_ID), AppController.class, "task"); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java index a8a756d305..0c6d7736fb 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java @@ -127,31 +127,28 @@ public class JobBlock extends HtmlBlock { th(_TH, "Running"). th(_TH, "Complete")._(). tr(_ODD). - th(). - a(url("tasks", jid, "m"), "Map")._(). + th("Map"). td(). div(_PROGRESSBAR). $title(join(jinfo.getMapProgressPercent(), '%')). // tooltip div(_PROGRESSBAR_VALUE). $style(join("width:", jinfo.getMapProgressPercent(), '%'))._()._()._(). - td(String.valueOf(jinfo.getMapsTotal())). - td(String.valueOf(jinfo.getMapsPending())). - td(String.valueOf(jinfo.getMapsRunning())). - td(String.valueOf(jinfo.getMapsCompleted()))._(). + td().a(url("tasks", jid, "m", "ALL"),String.valueOf(jinfo.getMapsTotal()))._(). + td().a(url("tasks", jid, "m", "PENDING"),String.valueOf(jinfo.getMapsPending()))._(). + td().a(url("tasks", jid, "m", "RUNNING"),String.valueOf(jinfo.getMapsRunning()))._(). + td().a(url("tasks", jid, "m", "COMPLETED"),String.valueOf(jinfo.getMapsCompleted()))._()._(). tr(_EVEN). - th(). - a(url("tasks", jid, "r"), "Reduce")._(). + th("Reduce"). td(). div(_PROGRESSBAR). $title(join(jinfo.getReduceProgressPercent(), '%')). // tooltip div(_PROGRESSBAR_VALUE). $style(join("width:", jinfo.getReduceProgressPercent(), '%'))._()._()._(). - td(String.valueOf(jinfo.getReducesTotal())). - td(String.valueOf(jinfo.getReducesPending())). - td(String.valueOf(jinfo.getReducesRunning())). - td(String.valueOf(jinfo.getReducesCompleted()))._() + td().a(url("tasks", jid, "r", "ALL"),String.valueOf(jinfo.getReducesTotal()))._(). + td().a(url("tasks", jid, "r", "PENDING"),String.valueOf(jinfo.getReducesPending()))._(). + td().a(url("tasks", jid, "r", "RUNNING"),String.valueOf(jinfo.getReducesRunning()))._(). + td().a(url("tasks", jid, "r", "COMPLETED"),String.valueOf(jinfo.getReducesCompleted()))._()._() ._(). - // Attempts table table("#job"). tr(). diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java index e2c65e320b..a2b19e4ae4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java @@ -18,6 +18,7 @@ package org.apache.hadoop.mapreduce.v2.app.webapp; +import static org.apache.hadoop.mapreduce.v2.app.webapp.AMParams.TASK_STATE; import static org.apache.hadoop.mapreduce.v2.app.webapp.AMParams.TASK_TYPE; import static org.apache.hadoop.yarn.util.StringHelper.join; import static org.apache.hadoop.yarn.util.StringHelper.percent; @@ -71,6 +72,25 @@ public class TasksBlock extends HtmlBlock { if (type != null && task.getType() != type) { continue; } + String taskStateStr = $(TASK_STATE); + if (taskStateStr == null || taskStateStr.trim().equals("")) { + taskStateStr = "ALL"; + } + + if (!taskStateStr.equalsIgnoreCase("ALL")) + { + try { + // get stateUI enum + MRApps.TaskStateUI stateUI = MRApps.taskState(taskStateStr); + if (!stateUI.correspondsTo(task.getState())) + { + continue; + } + } catch (IllegalArgumentException e) { + continue; // not supported state, ignore + } + } + TaskInfo info = new TaskInfo(task); String tid = info.getId(); String pct = percent(info.getProgress() / 100); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java index c998a8d4d0..500f21f62e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java @@ -48,6 +48,7 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState; import org.apache.hadoop.mapreduce.v2.api.records.TaskId; +import org.apache.hadoop.mapreduce.v2.api.records.TaskState; import org.apache.hadoop.mapreduce.v2.api.records.TaskType; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.ContainerLogAppender; @@ -125,6 +126,23 @@ public boolean correspondsTo(TaskAttemptState state) { } } + public static enum TaskStateUI { + RUNNING( + new TaskState[]{TaskState.RUNNING}), + PENDING(new TaskState[]{TaskState.SCHEDULED}), + COMPLETED(new TaskState[]{TaskState.SUCCEEDED, TaskState.FAILED, TaskState.KILLED}); + + private final List correspondingStates; + + private TaskStateUI(TaskState[] correspondingStates) { + this.correspondingStates = Arrays.asList(correspondingStates); + } + + public boolean correspondsTo(TaskState state) { + return this.correspondingStates.contains(state); + } + } + public static TaskType taskType(String symbol) { // JDK 7 supports switch on strings if (symbol.equals("m")) return TaskType.MAP; @@ -136,6 +154,10 @@ public static TaskAttemptStateUI taskAttemptState(String attemptStateStr) { return TaskAttemptStateUI.valueOf(attemptStateStr); } + public static TaskStateUI taskState(String taskStateStr) { + return TaskStateUI.valueOf(taskStateStr); + } + // gets the base name of the MapReduce framework or null if no // framework was configured private static String getMRFrameworkName(Configuration conf) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java index 9c36eebf27..4c8930e956 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java @@ -38,6 +38,7 @@ import org.apache.hadoop.mapreduce.v2.api.records.JobId; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId; import org.apache.hadoop.mapreduce.v2.api.records.TaskId; +import org.apache.hadoop.mapreduce.v2.api.records.TaskState; import org.apache.hadoop.mapreduce.v2.api.records.TaskType; import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.util.StringUtils; @@ -464,4 +465,13 @@ public void testLogSystemProperties() throws Exception { assertTrue(value.contains(os)); assertFalse(value.contains(version)); } + + @Test + public void testTaskStateUI() { + assertTrue(MRApps.TaskStateUI.PENDING.correspondsTo(TaskState.SCHEDULED)); + assertTrue(MRApps.TaskStateUI.COMPLETED.correspondsTo(TaskState.SUCCEEDED)); + assertTrue(MRApps.TaskStateUI.COMPLETED.correspondsTo(TaskState.FAILED)); + assertTrue(MRApps.TaskStateUI.COMPLETED.correspondsTo(TaskState.KILLED)); + assertTrue(MRApps.TaskStateUI.RUNNING.correspondsTo(TaskState.RUNNING)); + } }