From 89ded89e86e5d9a634d92a5d8a7c889744d97f94 Mon Sep 17 00:00:00 2001 From: Gera Shegalov Date: Tue, 21 Apr 2015 12:36:37 -0700 Subject: [PATCH] MAPREDUCE-6297. Task Id of the failed task in diagnostics should link to the task page. (Siqi Li via gera) --- hadoop-mapreduce-project/CHANGES.txt | 3 ++ .../org/apache/hadoop/mapreduce/TaskID.java | 35 +++++++------------ .../mapreduce/v2/hs/webapp/HsJobBlock.java | 8 ++++- .../mapreduce/v2/hs/webapp/TestBlocks.java | 20 ++++++++++- .../v2/hs/webapp/TestHsWebServicesTasks.java | 27 +++++++++----- 5 files changed, 60 insertions(+), 33 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 0cf5c4b4e2..ccdf6d6b67 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -340,6 +340,9 @@ Release 2.8.0 - UNRELEASED MAPREDUCE-6293. Set job classloader on uber-job's LocalContainerLauncher event thread. (Sangjin Lee via gera) + MAPREDUCE-6297. Task Id of the failed task in diagnostics should link to + the task page. (Siqi Li via gera) + Release 2.7.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskID.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskID.java index 488ffcc62e..b9817dd16e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskID.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskID.java @@ -25,6 +25,8 @@ import java.util.EnumMap; import java.util.HashMap; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -58,6 +60,9 @@ public class TaskID extends org.apache.hadoop.mapred.ID { protected static final String TASK = "task"; protected static final NumberFormat idFormat = NumberFormat.getInstance(); + public static final String TASK_ID_REGEX = TASK + "_(\\d+)_(\\d+)_" + + CharTaskTypeMaps.allTaskTypes + "_(\\d+)"; + public static final Pattern taskIdPattern = Pattern.compile(TASK_ID_REGEX); static { idFormat.setGroupingUsed(false); idFormat.setMinimumIntegerDigits(6); @@ -207,29 +212,15 @@ public static TaskID forName(String str) throws IllegalArgumentException { if(str == null) return null; - String exceptionMsg = null; - try { - String[] parts = str.split("_"); - if(parts.length == 5) { - if(parts[0].equals(TASK)) { - String type = parts[3]; - TaskType t = CharTaskTypeMaps.getTaskType(type.charAt(0)); - if(t != null) { - - return new org.apache.hadoop.mapred.TaskID(parts[1], - Integer.parseInt(parts[2]), - t, - Integer.parseInt(parts[4])); - } else - exceptionMsg = "Bad TaskType identifier. TaskId string : " + str - + " is not properly formed."; - } - } - }catch (Exception ex) {//fall below - } - if (exceptionMsg == null) { - exceptionMsg = "TaskId string : " + str + " is not properly formed"; + Matcher m = taskIdPattern.matcher(str); + if (m.matches()) { + return new org.apache.hadoop.mapred.TaskID(m.group(1), + Integer.parseInt(m.group(2)), + CharTaskTypeMaps.getTaskType(m.group(3).charAt(0)), + Integer.parseInt(m.group(4))); } + String exceptionMsg = "TaskId string : " + str + " is not properly formed" + + "\nReason: " + m.toString(); throw new IllegalArgumentException(exceptionMsg); } /** diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobBlock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobBlock.java index 4a13e0be72..f3341a6858 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobBlock.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobBlock.java @@ -27,6 +27,7 @@ import java.util.Date; import java.util.List; +import org.apache.hadoop.mapreduce.TaskID; import org.apache.hadoop.mapreduce.v2.api.records.AMInfo; import org.apache.hadoop.mapreduce.v2.api.records.JobId; import org.apache.hadoop.mapreduce.v2.app.AppContext; @@ -98,7 +99,7 @@ public class HsJobBlock extends HtmlBlock { if(diagnostics != null && !diagnostics.isEmpty()) { StringBuffer b = new StringBuffer(); for(String diag: diagnostics) { - b.append(diag); + b.append(addTaskLinks(diag)); } infoBlock._("Diagnostics:", b.toString()); } @@ -203,4 +204,9 @@ public class HsJobBlock extends HtmlBlock { _(). _(); } + + static String addTaskLinks(String text) { + return TaskID.taskIdPattern.matcher(text).replaceAll( + "$0"); + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestBlocks.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestBlocks.java index 723136769e..b82965a38e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestBlocks.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestBlocks.java @@ -65,6 +65,7 @@ import org.apache.hadoop.yarn.webapp.view.BlockForTest; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; import org.apache.hadoop.yarn.webapp.view.HtmlBlock.Block; +import org.junit.Assert; import org.junit.Test; import static org.junit.Assert.*; @@ -78,6 +79,23 @@ public class TestBlocks { private ByteArrayOutputStream data = new ByteArrayOutputStream(); + @Test + public void testPullTaskLink(){ + Task task = getTask(0); + String taskId = task.getID().toString(); + + Assert.assertEquals("pull links doesn't work correctly", + "Task failed " + + taskId + "" + , HsJobBlock.addTaskLinks("Task failed " + taskId)); + + Assert.assertEquals("pull links doesn't work correctly", + "Task failed " + + taskId + "\n Job failed as tasks failed. failedMaps:1 failedReduces:0" + , HsJobBlock.addTaskLinks("Task failed " + taskId + "\n " + + "Job failed as tasks failed. failedMaps:1 failedReduces:0")); + } + /** * test HsTasksBlock's rendering. */ @@ -241,7 +259,7 @@ public void testHsController() throws Exception { assertEquals(HsAttemptsPage.class, controller.attemptsPage()); controller.set(AMParams.JOB_ID, "job_01_01"); - controller.set(AMParams.TASK_ID, "task_01_01_m01_01"); + controller.set(AMParams.TASK_ID, "task_01_01_m_01"); controller.set(AMParams.TASK_TYPE, "m"); controller.set(AMParams.ATTEMPT_STATE, "State"); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java index ee0ccc6afc..22fa46a47c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java @@ -33,6 +33,7 @@ import javax.xml.parsers.DocumentBuilderFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.TaskID; import org.apache.hadoop.mapreduce.v2.api.records.JobId; import org.apache.hadoop.mapreduce.v2.api.records.TaskId; import org.apache.hadoop.mapreduce.v2.api.records.TaskReport; @@ -368,9 +369,11 @@ public void testTaskIdBogus() throws JSONException, Exception { String message = exception.getString("message"); String type = exception.getString("exception"); String classname = exception.getString("javaClassName"); - WebServicesTestUtils.checkStringMatch("exception message", + WebServicesTestUtils.checkStringEqual("exception message", "java.lang.Exception: TaskId string : " - + "bogustaskid is not properly formed", message); + + "bogustaskid is not properly formed" + + "\nReason: java.util.regex.Matcher[pattern=" + + TaskID.TASK_ID_REGEX + " region=0,11 lastmatch=]", message); WebServicesTestUtils.checkStringMatch("exception type", "NotFoundException", type); WebServicesTestUtils.checkStringMatch("exception classname", @@ -432,9 +435,11 @@ public void testTaskIdInvalid() throws JSONException, Exception { String message = exception.getString("message"); String type = exception.getString("exception"); String classname = exception.getString("javaClassName"); - WebServicesTestUtils.checkStringMatch("exception message", - "java.lang.Exception: Bad TaskType identifier. TaskId string : " - + "task_0_0000_d_000000 is not properly formed.", message); + WebServicesTestUtils.checkStringEqual("exception message", + "java.lang.Exception: TaskId string : " + + "task_0_0000_d_000000 is not properly formed" + + "\nReason: java.util.regex.Matcher[pattern=" + + TaskID.TASK_ID_REGEX + " region=0,20 lastmatch=]", message); WebServicesTestUtils.checkStringMatch("exception type", "NotFoundException", type); WebServicesTestUtils.checkStringMatch("exception classname", @@ -464,9 +469,11 @@ public void testTaskIdInvalid2() throws JSONException, Exception { String message = exception.getString("message"); String type = exception.getString("exception"); String classname = exception.getString("javaClassName"); - WebServicesTestUtils.checkStringMatch("exception message", + WebServicesTestUtils.checkStringEqual("exception message", "java.lang.Exception: TaskId string : " - + "task_0000_m_000000 is not properly formed", message); + + "task_0000_m_000000 is not properly formed" + + "\nReason: java.util.regex.Matcher[pattern=" + + TaskID.TASK_ID_REGEX + " region=0,18 lastmatch=]", message); WebServicesTestUtils.checkStringMatch("exception type", "NotFoundException", type); WebServicesTestUtils.checkStringMatch("exception classname", @@ -496,9 +503,11 @@ public void testTaskIdInvalid3() throws JSONException, Exception { String message = exception.getString("message"); String type = exception.getString("exception"); String classname = exception.getString("javaClassName"); - WebServicesTestUtils.checkStringMatch("exception message", + WebServicesTestUtils.checkStringEqual("exception message", "java.lang.Exception: TaskId string : " - + "task_0_0000_m is not properly formed", message); + + "task_0_0000_m is not properly formed" + + "\nReason: java.util.regex.Matcher[pattern=" + + TaskID.TASK_ID_REGEX + " region=0,13 lastmatch=]", message); WebServicesTestUtils.checkStringMatch("exception type", "NotFoundException", type); WebServicesTestUtils.checkStringMatch("exception classname",