From 159a715eef23046ee9909289defe62e68a5df533 Mon Sep 17 00:00:00 2001 From: Weiwei Yang Date: Wed, 13 Mar 2019 17:00:35 +0800 Subject: [PATCH] MAPREDUCE-7192. JobHistoryServer attempts page support jump to containers log page in NM when logAggregation is disable. Contributed by Jiandan Yang. --- .../v2/hs/webapp/HsAttemptsPage.java | 5 ++-- .../mapreduce/v2/hs/webapp/HsTaskPage.java | 24 +++++++++++---- .../mapreduce/v2/hs/webapp/TestBlocks.java | 30 +++++++++++++++++-- 3 files changed, 48 insertions(+), 11 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsAttemptsPage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsAttemptsPage.java index 9233fd3eef..b5bf719459 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsAttemptsPage.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsAttemptsPage.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Map; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId; import org.apache.hadoop.mapreduce.v2.api.records.TaskId; import org.apache.hadoop.mapreduce.v2.api.records.TaskType; @@ -46,8 +47,8 @@ public class HsAttemptsPage extends HsTaskPage { static class FewAttemptsBlock extends HsTaskPage.AttemptsBlock { @Inject - FewAttemptsBlock(App ctx) { - super(ctx); + FewAttemptsBlock(App ctx, Configuration conf) { + super(ctx, conf); } /* diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java index 5b0c59f51a..65fd1eac67 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsTaskPage.java @@ -30,6 +30,7 @@ import java.util.Collection; import org.apache.commons.text.StringEscapeUtils; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.v2.api.records.TaskId; import org.apache.hadoop.mapreduce.v2.api.records.TaskType; import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt; @@ -38,6 +39,7 @@ import org.apache.hadoop.mapreduce.v2.app.webapp.dao.TaskAttemptInfo; import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.mapreduce.v2.util.MRWebAppUtil; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.webapp.SubView; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet; @@ -61,10 +63,12 @@ public class HsTaskPage extends HsView { */ static class AttemptsBlock extends HtmlBlock { final App app; + final Configuration conf; @Inject - AttemptsBlock(App ctx) { + AttemptsBlock(App ctx, Configuration conf) { app = ctx; + this.conf = conf; } @Override @@ -152,13 +156,21 @@ protected void render(Block html) { StringEscapeUtils.escapeHtml4(ta.getStatus()))).append("\",\"") .append("") - .append(nodeRackName + "/" + nodeHttpAddr + "\",\"") + .append(nodeRackName + "/" + nodeHttpAddr + "\",\""); - .append("logs\",\"") + String logsUrl = url("logs", nodeIdString, containerIdString, taid, + app.getJob().getUserName()); + if (!conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, + YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLED)) { + logsUrl = + url(MRWebAppUtil.getYARNWebappScheme(), nodeHttpAddr, "node", + "containerlogs", containerIdString, + app.getJob().getUserName()); + } + attemptsTableData.append("logs\",\""); - .append(attemptStartTime).append("\",\""); + attemptsTableData.append(attemptStartTime).append("\",\""); if(type == TaskType.REDUCE) { attemptsTableData.append(shuffleFinishTime).append("\",\"") diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestBlocks.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestBlocks.java index 18687ff4fe..337f182c68 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestBlocks.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestBlocks.java @@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.webapp.Controller.RequestContext; import org.apache.hadoop.yarn.webapp.View.ViewContext; import org.apache.hadoop.yarn.webapp.Controller; @@ -193,7 +194,9 @@ public void testAttemptsBlock() { when(job.getUserName()).thenReturn("User"); app.setJob(job); - AttemptsBlockForTest block = new AttemptsBlockForTest(app); + Configuration conf = new Configuration(); + conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); + AttemptsBlockForTest block = new AttemptsBlockForTest(app, conf); block.addParameter(AMParams.TASK_TYPE, "r"); PrintWriter pWriter = new PrintWriter(data); @@ -212,6 +215,27 @@ public void testAttemptsBlock() { assertTrue(data.toString().contains("100010")); assertTrue(data.toString().contains("100011")); assertTrue(data.toString().contains("100012")); + data.reset(); + conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, false); + block = new AttemptsBlockForTest(app, conf); + block.addParameter(AMParams.TASK_TYPE, "r"); + + pWriter = new PrintWriter(data); + html = new BlockForTest(new HtmlBlockForTest(), pWriter, 0, false); + + block.render(html); + pWriter.flush(); + // should be printed information about attempts + assertTrue(data.toString().contains("attempt_0_0001_r_000000_0")); + assertTrue(data.toString().contains("SUCCEEDED")); + assertFalse(data.toString().contains("Processed 128/128 records

\n")); + assertTrue(data.toString().contains("Processed 128\\/128 records <p> \\n")); + assertTrue(data.toString().contains( + "Node address:node:containerlogs:container_0_0005_01_000001:User:")); + assertTrue(data.toString().contains("100002")); + assertTrue(data.toString().contains("100010")); + assertTrue(data.toString().contains("100011")); + assertTrue(data.toString().contains("100012")); } /** @@ -438,8 +462,8 @@ public void addParameter(String name, String value) { return value == null ? defaultValue : value; } - public AttemptsBlockForTest(App ctx) { - super(ctx); + public AttemptsBlockForTest(App ctx, Configuration conf) { + super(ctx, conf); } @Override