From 9445abb500530d5988346c2c8515f5f85553301e Mon Sep 17 00:00:00 2001 From: Prabhu Josephraj Date: Tue, 8 Jun 2021 20:37:40 +0530 Subject: [PATCH] YARN-10792. Set Completed AppAttempt LogsLink to Log Server URL. Contributed by Abhinaba Sarkar --- .../webapp/dao/AppAttemptInfo.java | 26 +++++++++--- .../webapp/TestRMWebServicesAppAttempts.java | 40 +++++++++++++++++++ 2 files changed, 61 insertions(+), 5 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java index 00454afe0e..499d4e1937 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java @@ -23,7 +23,9 @@ import com.google.gson.Gson; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; @@ -31,6 +33,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import static org.apache.hadoop.yarn.util.StringHelper.PATH_JOINER; + @XmlRootElement(name = "appAttempt") @XmlAccessorType(XmlAccessType.FIELD) public class AppAttemptInfo { @@ -64,15 +68,29 @@ public AppAttemptInfo(ResourceManager rm, RMAppAttempt attempt, this.id = attempt.getAppAttemptId().getAttemptId(); this.startTime = attempt.getStartTime(); this.finishedTime = attempt.getFinishTime(); + this.appAttemptState = attempt.getAppAttemptState(); + this.appAttemptId = attempt.getAppAttemptId().toString(); Container masterContainer = attempt.getMasterContainer(); if (masterContainer != null && hasAccess) { this.containerId = masterContainer.getId().toString(); this.nodeHttpAddress = masterContainer.getNodeHttpAddress(); this.nodeId = masterContainer.getNodeId().toString(); - this.logsLink = WebAppUtils.getRunningLogURL(schemePrefix - + masterContainer.getNodeHttpAddress(), - masterContainer.getId().toString(), user); + Configuration conf = rm.getRMContext().getYarnConfiguration(); + String logServerUrl = conf.get(YarnConfiguration.YARN_LOG_SERVER_URL); + if ((this.appAttemptState == RMAppAttemptState.FAILED || + this.appAttemptState == RMAppAttemptState.FINISHED || + this.appAttemptState == RMAppAttemptState.KILLED) && + logServerUrl != null) { + this.logsLink = PATH_JOINER.join(logServerUrl, + masterContainer.getNodeId().toString(), + masterContainer.getId().toString(), + masterContainer.getId().toString(), user); + } else { + this.logsLink = WebAppUtils.getRunningLogURL(schemePrefix + + masterContainer.getNodeHttpAddress(), + masterContainer.getId().toString(), user); + } Gson gson = new Gson(); this.exportPorts = gson.toJson(masterContainer.getExposedPorts()); @@ -90,8 +108,6 @@ public AppAttemptInfo(ResourceManager rm, RMAppAttempt attempt, } } } - this.appAttemptId = attempt.getAppAttemptId().toString(); - this.appAttemptState = attempt.getAppAttemptState(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppAttempts.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppAttempts.java index df168f757b..102f13897f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppAttempts.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppAttempts.java @@ -60,6 +60,7 @@ import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseStatusCode; import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.checkStringMatch; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -124,6 +125,45 @@ public void testAppAttempts() throws Exception { rm.stop(); } + @Test (timeout = 20000) + public void testCompletedAppAttempt() throws Exception { + Configuration conf = rm.getConfig(); + String logServerUrl = "http://localhost:19888/jobhistory/logs"; + conf.set(YarnConfiguration.YARN_LOG_SERVER_URL, logServerUrl); + conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1); + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 8192); + MockRMAppSubmissionData data = + MockRMAppSubmissionData.Builder.createWithMemory(CONTAINER_MB, rm) + .withAppName("testwordcount") + .withUser("user1") + .build(); + RMApp app1 = MockRMAppSubmitter.submit(rm, data); + MockAM am = MockRM.launchAndRegisterAM(app1, rm, amNodeManager); + // fail the AM by sending CONTAINER_FINISHED event without registering. + amNodeManager.nodeHeartbeat(am.getApplicationAttemptId(), 1, + ContainerState.COMPLETE); + rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FAILED); + rm.waitForState(app1.getApplicationId(), RMAppState.FAILED); + + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").path(app1.getApplicationId().toString()) + .path("appattempts").accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + JSONObject json = response.getEntity(JSONObject.class); + JSONObject jsonAppAttempts = json.getJSONObject("appAttempts"); + JSONArray jsonArray = jsonAppAttempts.getJSONArray("appAttempt"); + JSONObject info = jsonArray.getJSONObject(0); + String logsLink = info.getString("logsLink"); + String containerId = app1.getCurrentAppAttempt().getMasterContainer() + .getId().toString(); + assertThat(logsLink).isEqualTo(logServerUrl + + "/127.0.0.1:1234/" + containerId + "/" + containerId + "/" + + "user1"); + rm.stop(); + } + @Test (timeout = 20000) public void testMultipleAppAttempts() throws Exception { rm.start();