big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-5553. Allow users to easily access completed/pending/successful/failed tasks on MR AM web-ui. Contributed by Paul Han.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1576946 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Arun Murthy 2014-03-12 22:04:48 +00:00
parent cbf1c72d1b
commit 3bbd67173e
7 changed files with 68 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-mapreduce-project/CHANGES.txt
View File

@ -188,6 +188,10 @@ Release 2.4.0 - UNRELEASED
MAPREDUCE-5773. Provide dedicated MRAppMaster syslog length limit (Gera
Shegalov via jlowe)
MAPREDUCE-5553. Allow users to easily access
completed/pending/successful/failed tasks on MR AM web-ui. (Paul Han via
acmurthy)
OPTIMIZATIONS
BUG FIXES

1
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMParams.java
View File

@ -27,6 +27,7 @@ public interface AMParams {
static final String JOB_ID = "job.id";
static final String TASK_ID = "task.id";
static final String TASK_TYPE = "task.type";
static final String TASK_STATE = "task.state";
static final String ATTEMPT_STATE = "attempt.state";
static final String COUNTER_GROUP = "counter.group";
static final String COUNTER_NAME = "counter.name";

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebApp.java
View File

@ -40,7 +40,7 @@ public void setup() {
route(pajoin("/jobcounters", JOB_ID), AppController.class, "jobCounters");
route(pajoin("/singlejobcounter",JOB_ID, COUNTER_GROUP, COUNTER_NAME),
AppController.class, "singleJobCounter");
route(pajoin("/tasks", JOB_ID, TASK_TYPE), AppController.class, "tasks");
route(pajoin("/tasks", JOB_ID, TASK_TYPE, TASK_STATE), AppController.class, "tasks");
route(pajoin("/attempts", JOB_ID, TASK_TYPE, ATTEMPT_STATE),
AppController.class, "attempts");
route(pajoin("/task", TASK_ID), AppController.class, "task");

23
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java
View File

@ -127,31 +127,28 @@ public class JobBlock extends HtmlBlock {
th(_TH, "Running").
th(_TH, "Complete")._().
tr(_ODD).
th().
a(url("tasks", jid, "m"), "Map")._().
th("Map").
td().
div(_PROGRESSBAR).
$title(join(jinfo.getMapProgressPercent(), '%')). // tooltip
div(_PROGRESSBAR_VALUE).
$style(join("width:", jinfo.getMapProgressPercent(), '%'))._()._()._().
td(String.valueOf(jinfo.getMapsTotal())).
td(String.valueOf(jinfo.getMapsPending())).
td(String.valueOf(jinfo.getMapsRunning())).
td(String.valueOf(jinfo.getMapsCompleted()))._().
td().a(url("tasks", jid, "m", "ALL"),String.valueOf(jinfo.getMapsTotal()))._().
td().a(url("tasks", jid, "m", "PENDING"),String.valueOf(jinfo.getMapsPending()))._().
td().a(url("tasks", jid, "m", "RUNNING"),String.valueOf(jinfo.getMapsRunning()))._().
td().a(url("tasks", jid, "m", "COMPLETED"),String.valueOf(jinfo.getMapsCompleted()))._()._().
tr(_EVEN).
th().
a(url("tasks", jid, "r"), "Reduce")._().
th("Reduce").
td().
div(_PROGRESSBAR).
$title(join(jinfo.getReduceProgressPercent(), '%')). // tooltip
div(_PROGRESSBAR_VALUE).
$style(join("width:", jinfo.getReduceProgressPercent(), '%'))._()._()._().
td(String.valueOf(jinfo.getReducesTotal())).
td(String.valueOf(jinfo.getReducesPending())).
td(String.valueOf(jinfo.getReducesRunning())).
td(String.valueOf(jinfo.getReducesCompleted()))._()
td().a(url("tasks", jid, "r", "ALL"),String.valueOf(jinfo.getReducesTotal()))._().
td().a(url("tasks", jid, "r", "PENDING"),String.valueOf(jinfo.getReducesPending()))._().
td().a(url("tasks", jid, "r", "RUNNING"),String.valueOf(jinfo.getReducesRunning()))._().
td().a(url("tasks", jid, "r", "COMPLETED"),String.valueOf(jinfo.getReducesCompleted()))._()._()
._().
// Attempts table
table("#job").
tr().

20
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TasksBlock.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.mapreduce.v2.app.webapp;
import static org.apache.hadoop.mapreduce.v2.app.webapp.AMParams.TASK_STATE;
import static org.apache.hadoop.mapreduce.v2.app.webapp.AMParams.TASK_TYPE;
import static org.apache.hadoop.yarn.util.StringHelper.join;
import static org.apache.hadoop.yarn.util.StringHelper.percent;
@ -71,6 +72,25 @@ public class TasksBlock extends HtmlBlock {
if (type != null && task.getType() != type) {
continue;
}
String taskStateStr = $(TASK_STATE);
if (taskStateStr == null || taskStateStr.trim().equals("")) {
taskStateStr = "ALL";
}
if (!taskStateStr.equalsIgnoreCase("ALL"))
{
try {
// get stateUI enum
MRApps.TaskStateUI stateUI = MRApps.taskState(taskStateStr);
if (!stateUI.correspondsTo(task.getState()))
{
continue;
}
} catch (IllegalArgumentException e) {
continue; // not supported state, ignore
}
}
TaskInfo info = new TaskInfo(task);
String tid = info.getId();
String pct = percent(info.getProgress() / 100);

22
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
View File

@ -48,6 +48,7 @@
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.ContainerLogAppender;
@ -125,6 +126,23 @@ public boolean correspondsTo(TaskAttemptState state) {
}
}
public static enum TaskStateUI {
RUNNING(
new TaskState[]{TaskState.RUNNING}),
PENDING(new TaskState[]{TaskState.SCHEDULED}),
COMPLETED(new TaskState[]{TaskState.SUCCEEDED, TaskState.FAILED, TaskState.KILLED});
private final List<TaskState> correspondingStates;
private TaskStateUI(TaskState[] correspondingStates) {
this.correspondingStates = Arrays.asList(correspondingStates);
}
public boolean correspondsTo(TaskState state) {
return this.correspondingStates.contains(state);
}
}
public static TaskType taskType(String symbol) {
// JDK 7 supports switch on strings
if (symbol.equals("m")) return TaskType.MAP;
@ -136,6 +154,10 @@ public static TaskAttemptStateUI taskAttemptState(String attemptStateStr) {
return TaskAttemptStateUI.valueOf(attemptStateStr);
}
public static TaskStateUI taskState(String taskStateStr) {
return TaskStateUI.valueOf(taskStateStr);
}
// gets the base name of the MapReduce framework or null if no
// framework was configured
private static String getMRFrameworkName(Configuration conf) {

10
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
View File

@ -38,6 +38,7 @@
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.util.StringUtils;
@ -464,4 +465,13 @@ public void testLogSystemProperties() throws Exception {
assertTrue(value.contains(os));
assertFalse(value.contains(version));
}
@Test
public void testTaskStateUI() {
assertTrue(MRApps.TaskStateUI.PENDING.correspondsTo(TaskState.SCHEDULED));
assertTrue(MRApps.TaskStateUI.COMPLETED.correspondsTo(TaskState.SUCCEEDED));
assertTrue(MRApps.TaskStateUI.COMPLETED.correspondsTo(TaskState.FAILED));
assertTrue(MRApps.TaskStateUI.COMPLETED.correspondsTo(TaskState.KILLED));
assertTrue(MRApps.TaskStateUI.RUNNING.correspondsTo(TaskState.RUNNING));
}
}