big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10792. Set Completed AppAttempt LogsLink to Log Server URL. Contributed by Abhinaba Sarkar

Browse Source
This commit is contained in:
Prabhu Josephraj 2021-06-08 20:37:40 +05:30
parent ec16b1d3b9
commit 9445abb500
2 changed files with 61 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java
View File

@ -23,7 +23,9 @@
import com.google.gson.Gson;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
@ -31,6 +33,8 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import static org.apache.hadoop.yarn.util.StringHelper.PATH_JOINER;
@XmlRootElement(name = "appAttempt")
@XmlAccessorType(XmlAccessType.FIELD)
public class AppAttemptInfo {
@ -64,15 +68,29 @@ public AppAttemptInfo(ResourceManager rm, RMAppAttempt attempt,
this.id = attempt.getAppAttemptId().getAttemptId();
this.startTime = attempt.getStartTime();
this.finishedTime = attempt.getFinishTime();
this.appAttemptState = attempt.getAppAttemptState();
this.appAttemptId = attempt.getAppAttemptId().toString();
Container masterContainer = attempt.getMasterContainer();
if (masterContainer != null && hasAccess) {
this.containerId = masterContainer.getId().toString();
this.nodeHttpAddress = masterContainer.getNodeHttpAddress();
this.nodeId = masterContainer.getNodeId().toString();
this.logsLink = WebAppUtils.getRunningLogURL(schemePrefix
+ masterContainer.getNodeHttpAddress(),
masterContainer.getId().toString(), user);
Configuration conf = rm.getRMContext().getYarnConfiguration();
String logServerUrl = conf.get(YarnConfiguration.YARN_LOG_SERVER_URL);
if ((this.appAttemptState == RMAppAttemptState.FAILED ||
this.appAttemptState == RMAppAttemptState.FINISHED ||
this.appAttemptState == RMAppAttemptState.KILLED) &&
logServerUrl != null) {
this.logsLink = PATH_JOINER.join(logServerUrl,
masterContainer.getNodeId().toString(),
masterContainer.getId().toString(),
masterContainer.getId().toString(), user);
} else {
this.logsLink = WebAppUtils.getRunningLogURL(schemePrefix
+ masterContainer.getNodeHttpAddress(),
masterContainer.getId().toString(), user);
}
Gson gson = new Gson();
this.exportPorts = gson.toJson(masterContainer.getExposedPorts());
@ -90,8 +108,6 @@ public AppAttemptInfo(ResourceManager rm, RMAppAttempt attempt,
}
}
}
this.appAttemptId = attempt.getAppAttemptId().toString();
this.appAttemptState = attempt.getAppAttemptState();
}
}

40
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppAttempts.java
View File

@ -60,6 +60,7 @@
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseStatusCode;
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.checkStringMatch;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@ -124,6 +125,45 @@ public void testAppAttempts() throws Exception {
rm.stop();
}
@Test (timeout = 20000)
public void testCompletedAppAttempt() throws Exception {
Configuration conf = rm.getConfig();
String logServerUrl = "http://localhost:19888/jobhistory/logs";
conf.set(YarnConfiguration.YARN_LOG_SERVER_URL, logServerUrl);
conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 8192);
MockRMAppSubmissionData data =
MockRMAppSubmissionData.Builder.createWithMemory(CONTAINER_MB, rm)
.withAppName("testwordcount")
.withUser("user1")
.build();
RMApp app1 = MockRMAppSubmitter.submit(rm, data);
MockAM am = MockRM.launchAndRegisterAM(app1, rm, amNodeManager);
// fail the AM by sending CONTAINER_FINISHED event without registering.
amNodeManager.nodeHeartbeat(am.getApplicationAttemptId(), 1,
ContainerState.COMPLETE);
rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FAILED);
rm.waitForState(app1.getApplicationId(), RMAppState.FAILED);
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("apps").path(app1.getApplicationId().toString())
.path("appattempts").accept(MediaType.APPLICATION_JSON)
.get(ClientResponse.class);
JSONObject json = response.getEntity(JSONObject.class);
JSONObject jsonAppAttempts = json.getJSONObject("appAttempts");
JSONArray jsonArray = jsonAppAttempts.getJSONArray("appAttempt");
JSONObject info = jsonArray.getJSONObject(0);
String logsLink = info.getString("logsLink");
String containerId = app1.getCurrentAppAttempt().getMasterContainer()
.getId().toString();
assertThat(logsLink).isEqualTo(logServerUrl
+ "/127.0.0.1:1234/" + containerId + "/" + containerId + "/"
+ "user1");
rm.stop();
}
@Test (timeout = 20000)
public void testMultipleAppAttempts() throws Exception {
rm.start();