YARN-4701. When task logs are not available, port 8041 is referenced instead of port 8042 (haibochen via rkanter)

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/AMAttemptInfo.java

@@ -68,8 +68,8 @@ public AMAttemptInfo(AMInfo amInfo, String jobId, String user, String host,
     if (containerId != null) {
       this.containerId = containerId.toString();
       this.logsLink = join(host, pathPrefix,
-          ujoin("logs", this.nodeId, this.containerId, jobId, user));
+          ujoin("logs", this.nodeHttpAddress, this.containerId, jobId, user));
-      this.shortLogsLink = ujoin("logs", this.nodeId, this.containerId,
+      this.shortLogsLink = ujoin("logs", this.nodeHttpAddress, this.containerId,
           jobId, user);
     }
   }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java

@@ -766,7 +766,7 @@ public void verifyHsJobAttempts(JSONObject info, Job job)
     for (int i = 0; i < attempts.length(); i++) {
       JSONObject attempt = attempts.getJSONObject(i);
       verifyHsJobAttemptsGeneric(job, attempt.getString("nodeHttpAddress"),
-          attempt.getString("nodeId"), attempt.getInt("id"),
+          attempt.getInt("id"),
           attempt.getLong("startTime"), attempt.getString("containerId"),
           attempt.getString("logsLink"));
     }
@@ -779,7 +779,6 @@ public void verifyHsJobAttemptsXML(NodeList nodes, Job job) {
       Element element = (Element) nodes.item(i);
       verifyHsJobAttemptsGeneric(job,
           WebServicesTestUtils.getXmlString(element, "nodeHttpAddress"),
-          WebServicesTestUtils.getXmlString(element, "nodeId"),
           WebServicesTestUtils.getXmlInt(element, "id"),
           WebServicesTestUtils.getXmlLong(element, "startTime"),
           WebServicesTestUtils.getXmlString(element, "containerId"),
@@ -788,7 +787,7 @@ public void verifyHsJobAttemptsXML(NodeList nodes, Job job) {
   }
 
   public void verifyHsJobAttemptsGeneric(Job job, String nodeHttpAddress,
-      String nodeId, int id, long startTime, String containerId, String logsLink) {
+      int id, long startTime, String containerId, String logsLink) {
     boolean attemptFound = false;
     for (AMInfo amInfo : job.getAMInfos()) {
       if (amInfo.getAppAttemptId().getAttemptId() == id) {
@@ -798,16 +797,14 @@ public void verifyHsJobAttemptsGeneric(Job job, String nodeHttpAddress,
         int nmPort = amInfo.getNodeManagerPort();
         WebServicesTestUtils.checkStringMatch("nodeHttpAddress", nmHost + ":"
             + nmHttpPort, nodeHttpAddress);
-        WebServicesTestUtils.checkStringMatch("nodeId",
-            NodeId.newInstance(nmHost, nmPort).toString(), nodeId);
         assertTrue("startime not greater than 0", startTime > 0);
         WebServicesTestUtils.checkStringMatch("containerId", amInfo
             .getContainerId().toString(), containerId);
 
         String localLogsLink = join(
             "hsmockwebapp",
-            ujoin("logs", nodeId, containerId, MRApps.toString(job.getID()),
+            ujoin("logs", nodeHttpAddress, containerId,
-                job.getUserName()));
+              MRApps.toString(job.getID()), job.getUserName()));
 
         assertTrue("logsLink", logsLink.contains(localLogsLink));
       }

hadoop-yarn-project/CHANGES.txt

@@ -230,6 +230,9 @@ Release 2.9.0 - UNRELEASED
     YARN-4729. SchedulerApplicationAttempt#getTotalRequiredResources can throw 
     an NPE. (kasha)
 
+    YARN-4701. When task logs are not available, port 8041 is referenced
+    instead of port 8042 (haibochen via rkanter)
+
 Release 2.8.0 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java

@@ -80,11 +80,17 @@ protected void render(Block html) {
       logEntity = containerId.toString();
     }
 
+    String nmApplicationLogUrl = getApplicationLogURL(applicationId);
     if (!conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED,
         YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLED)) {
       html.h1()
           ._("Aggregation is not enabled. Try the nodemanager at " + nodeId)
           ._();
+      if(nmApplicationLogUrl != null) {
+        html.h1()
+            ._("Or see application log at " + nmApplicationLogUrl)
+            ._();
+      }
       return;
     }
 
@@ -107,6 +113,11 @@ protected void render(Block html) {
           ._("Logs not available for " + logEntity
               + ". Aggregation may not be complete, "
               + "Check back later or try the nodemanager at " + nodeId)._();
+      if(nmApplicationLogUrl != null)  {
+        html.h1()
+            ._("Or see application log at " + nmApplicationLogUrl)
+            ._();
+      }
       return;
     } catch (Exception ex) {
       html.h1()
@@ -353,4 +364,20 @@ private LogLimits verifyAndGetLogLimits(Block html) {
     limits.end = end;
     return limits;
   }
+
+  private String getApplicationLogURL(ApplicationId applicationId) {
+    String appId = applicationId.toString();
+    if (appId == null || appId.isEmpty()) {
+      return null;
+    }
+    String nodeId = $(NM_NODENAME);
+    if(nodeId == null || nodeId.isEmpty()) {
+      return null;
+    }
+    StringBuilder sb = new StringBuilder();
+    String scheme = YarnConfiguration.useHttps(this.conf) ? "https://":
+        "http://";
+    sb.append(scheme).append(nodeId).append("/node/application/").append(appId);
+    return sb.toString();
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogsBlock.java

@@ -88,6 +88,26 @@ public void testAccessDenied() throws Exception {
 
   }
 
+  @Test
+  public void testBlockContainsPortNumForUnavailableAppLog() {
+    FileUtil.fullyDelete(new File("target/logs"));
+    Configuration configuration = getConfiguration();
+
+    String nodeName = configuration.get(YarnConfiguration.NM_WEBAPP_ADDRESS,
+        YarnConfiguration.DEFAULT_NM_WEBAPP_ADDRESS);
+    AggregatedLogsBlockForTest aggregatedBlock = getAggregatedLogsBlockForTest(
+        configuration, "admin", "container_0_0001_01_000001", nodeName);
+    ByteArrayOutputStream data = new ByteArrayOutputStream();
+    PrintWriter printWriter = new PrintWriter(data);
+    HtmlBlock html = new HtmlBlockForTest();
+    HtmlBlock.Block block = new BlockForTest(html, printWriter, 10, false);
+    aggregatedBlock.render(block);
+
+    block.getWriter().flush();
+    String out = data.toString();
+    assertTrue(out.contains(nodeName));
+  }
+
   /**
    * try to read bad logs
    *