YARN-1884. Added nodeHttpAddress into ContainerReport and fixed the link to NM web page. Contributed by Xuan Gong.

This commit is contained in:
Zhijie Shen 2015-03-11 19:35:19 -07:00
parent 7a346bcb4f
commit 85f6d67fa7
21 changed files with 147 additions and 38 deletions

View File

@ -752,6 +752,9 @@ Release 2.7.0 - UNRELEASED
YARN-3295. Fix documentation nits found in markdown conversion.
(Masatake Iwasaki via ozawa)
YARN-1884. Added nodeHttpAddress into ContainerReport and fixed the link to NM
web page. (Xuan Gong via zjshen)
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES

View File

@ -41,6 +41,7 @@
* <li>{@link ContainerState} of the container.</li>
* <li>Diagnostic information in case of errors.</li>
* <li>Log URL.</li>
* <li>nodeHttpAddress</li>
* </ul>
* </p>
*
@ -54,7 +55,8 @@ public abstract class ContainerReport {
public static ContainerReport newInstance(ContainerId containerId,
Resource allocatedResource, NodeId assignedNode, Priority priority,
long creationTime, long finishTime, String diagnosticInfo, String logUrl,
int containerExitStatus, ContainerState containerState) {
int containerExitStatus, ContainerState containerState,
String nodeHttpAddress) {
ContainerReport report = Records.newRecord(ContainerReport.class);
report.setContainerId(containerId);
report.setAllocatedResource(allocatedResource);
@ -66,6 +68,7 @@ public static ContainerReport newInstance(ContainerId containerId,
report.setLogUrl(logUrl);
report.setContainerExitStatus(containerExitStatus);
report.setContainerState(containerState);
report.setNodeHttpAddress(nodeHttpAddress);
return report;
}
@ -199,4 +202,16 @@ public static ContainerReport newInstance(ContainerId containerId,
@Unstable
public abstract void setContainerExitStatus(int containerExitStatus);
/**
* Get the Node Http address of the container
*
* @return the node http address of the container
*/
@Public
@Unstable
public abstract String getNodeHttpAddress();
@Private
@Unstable
public abstract void setNodeHttpAddress(String nodeHttpAddress);
}

View File

@ -98,6 +98,7 @@ message ContainerReportProto {
optional string log_url = 8;
optional int32 container_exit_status = 9;
optional ContainerStateProto container_state = 10;
optional string node_http_address = 11;
}
enum YarnApplicationStateProto {

View File

@ -63,7 +63,7 @@ public class ApplicationCLI extends YarnCLI {
"%30s\t%20s\t%35s\t%35s"
+ System.getProperty("line.separator");
private static final String CONTAINER_PATTERN =
"%30s\t%20s\t%20s\t%20s\t%20s\t%35s"
"%30s\t%20s\t%20s\t%20s\t%20s\t%20s\t%35s"
+ System.getProperty("line.separator");
private static final String APP_TYPE_CMD = "appTypes";
@ -355,6 +355,9 @@ private int printContainerReport(String containerId) throws YarnException,
containerReportStr.println(containerReport.getLogUrl());
containerReportStr.print("\tHost : ");
containerReportStr.println(containerReport.getAssignedNode());
containerReportStr.print("\tNodeHttpAddress : ");
containerReportStr.println(containerReport.getNodeHttpAddress() == null
? "N/A" : containerReport.getNodeHttpAddress());
containerReportStr.print("\tDiagnostics : ");
containerReportStr.print(containerReport.getDiagnosticsInfo());
} else {
@ -595,7 +598,7 @@ private void listContainers(String appAttemptId) throws YarnException,
.getContainers(ConverterUtils.toApplicationAttemptId(appAttemptId));
writer.println("Total number of containers " + ":" + appsReport.size());
writer.printf(CONTAINER_PATTERN, "Container-Id", "Start Time",
"Finish Time", "State", "Host", "LOG-URL");
"Finish Time", "State", "Host", "Node Http Address", "LOG-URL");
for (ContainerReport containerReport : appsReport) {
writer.printf(
CONTAINER_PATTERN,
@ -603,7 +606,9 @@ private void listContainers(String appAttemptId) throws YarnException,
Times.format(containerReport.getCreationTime()),
Times.format(containerReport.getFinishTime()),
containerReport.getContainerState(), containerReport
.getAssignedNode(), containerReport.getLogUrl());
.getAssignedNode(), containerReport.getNodeHttpAddress() == null
? "N/A" : containerReport.getNodeHttpAddress(),
containerReport.getLogUrl());
}
writer.flush();
}

View File

@ -77,7 +77,6 @@
import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse;
import org.apache.hadoop.yarn.api.records.AMCommand;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@ -715,7 +714,8 @@ public ApplicationAttemptReport createFakeApplicationAttemptReport() {
public ContainerReport createFakeContainerReport() {
return ContainerReport.newInstance(createFakeContainerId(), null,
NodeId.newInstance("localhost", 0), null, 1000l, 1200l, "", "", 0,
ContainerState.COMPLETE);
ContainerState.COMPLETE,
"http://" + NodeId.newInstance("localhost", 0).toString());
}
public List<ContainerReport> createFakeContainerReports() {

View File

@ -371,14 +371,16 @@ private void createAppReports() {
ContainerReport.newInstance(
ContainerId.newContainerId(attempt.getApplicationAttemptId(), 1),
null, NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234,
5678, "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
5678, "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE,
"http://" + NodeId.newInstance("host", 2345).toString());
containerReports.add(container);
ContainerReport container1 =
ContainerReport.newInstance(
ContainerId.newContainerId(attempt.getApplicationAttemptId(), 2),
null, NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234,
5678, "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
5678, "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE,
"http://" + NodeId.newInstance("host", 2345).toString());
containerReports.add(container1);
containers.put(attempt.getApplicationAttemptId(), containerReports);

View File

@ -117,7 +117,6 @@
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.UTCClock;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@ -613,13 +612,15 @@ private List<ApplicationReport> createAppReports() {
ContainerReport container = ContainerReport.newInstance(
ContainerId.newContainerId(attempt.getApplicationAttemptId(), 1), null,
NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
"diagnosticInfo", "logURL", 0, ContainerState.RUNNING);
"diagnosticInfo", "logURL", 0, ContainerState.RUNNING,
"http://" + NodeId.newInstance("host", 2345).toString());
containerReports.add(container);
ContainerReport container1 = ContainerReport.newInstance(
ContainerId.newContainerId(attempt.getApplicationAttemptId(), 2), null,
NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
"diagnosticInfo", "logURL", 0, ContainerState.RUNNING);
"diagnosticInfo", "logURL", 0, ContainerState.RUNNING,
"http://" + NodeId.newInstance("host", 2345).toString());
containerReports.add(container1);
containers.put(attempt.getApplicationAttemptId(), containerReports);
@ -630,18 +631,21 @@ private List<ApplicationReport> createAppReports() {
container = ContainerReport.newInstance(
ContainerId.newContainerId(attempt.getApplicationAttemptId(), 1), null,
NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
"diagnosticInfo", "logURL", 0, null);
"diagnosticInfo", "logURL", 0, null,
"http://" + NodeId.newInstance("host", 2345).toString());
containerReportsForAHS.add(container);
container1 = ContainerReport.newInstance(
ContainerId.newContainerId(attempt.getApplicationAttemptId(), 2), null,
NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
"diagnosticInfo", "HSlogURL", 0, null);
"diagnosticInfo", "HSlogURL", 0, null,
"http://" + NodeId.newInstance("host", 2345).toString());
containerReportsForAHS.add(container1);
ContainerReport container2 = ContainerReport.newInstance(
ContainerId.newContainerId(attempt.getApplicationAttemptId(),3), null,
NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
"diagnosticInfo", "HSlogURL", 0, ContainerState.COMPLETE);
"diagnosticInfo", "HSlogURL", 0, ContainerState.COMPLETE,
"http://" + NodeId.newInstance("host", 2345).toString());
containerReportsForAHS.add(container2);
containersFromAHS.put(attempt.getApplicationAttemptId(), containerReportsForAHS);

View File

@ -224,7 +224,8 @@ public void testGetContainerReport() throws Exception {
ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
ContainerReport container = ContainerReport.newInstance(containerId, null,
NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
"diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
"diagnosticInfo", "logURL", 0, ContainerState.COMPLETE,
"http://" + NodeId.newInstance("host", 2345).toString());
when(client.getContainerReport(any(ContainerId.class))).thenReturn(
container);
int result = cli.run(new String[] { "container", "-status",
@ -240,6 +241,7 @@ public void testGetContainerReport() throws Exception {
pw.println("\tState : COMPLETE");
pw.println("\tLOG-URL : logURL");
pw.println("\tHost : host:1234");
pw.println("\tNodeHttpAddress : http://host:2345");
pw.println("\tDiagnostics : diagnosticInfo");
pw.close();
String appReportStr = baos.toString("UTF-8");
@ -259,13 +261,16 @@ public void testGetContainers() throws Exception {
long time1=1234,time2=5678;
ContainerReport container = ContainerReport.newInstance(containerId, null,
NodeId.newInstance("host", 1234), Priority.UNDEFINED, time1, time2,
"diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
"diagnosticInfo", "logURL", 0, ContainerState.COMPLETE,
"http://" + NodeId.newInstance("host", 2345).toString());
ContainerReport container1 = ContainerReport.newInstance(containerId1, null,
NodeId.newInstance("host", 1234), Priority.UNDEFINED, time1, time2,
"diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
"diagnosticInfo", "logURL", 0, ContainerState.COMPLETE,
"http://" + NodeId.newInstance("host", 2345).toString());
ContainerReport container2 = ContainerReport.newInstance(containerId2, null,
NodeId.newInstance("host", 1234), Priority.UNDEFINED, time1,0,
"diagnosticInfo", "", 0, ContainerState.RUNNING);
"diagnosticInfo", "", 0, ContainerState.RUNNING,
"http://" + NodeId.newInstance("host", 2345).toString());
List<ContainerReport> reports = new ArrayList<ContainerReport>();
reports.add(container);
reports.add(container1);
@ -273,6 +278,7 @@ public void testGetContainers() throws Exception {
DateFormat dateFormat=new SimpleDateFormat("EEE MMM dd HH:mm:ss Z yyyy");
when(client.getContainers(any(ApplicationAttemptId.class))).thenReturn(
reports);
sysOutStream.reset();
int result = cli.run(new String[] { "container", "-list",
attemptId.toString() });
assertEquals(0, result);
@ -285,24 +291,28 @@ public void testGetContainers() throws Exception {
pw.print("\t Finish Time");
pw.print("\t State");
pw.print("\t Host");
pw.print("\t Node Http Address");
pw.println("\t LOG-URL");
pw.print(" container_1234_0005_01_000001");
pw.print("\t"+dateFormat.format(new Date(time1)));
pw.print("\t"+dateFormat.format(new Date(time2)));
pw.print("\t COMPLETE");
pw.print("\t host:1234");
pw.print("\t http://host:2345");
pw.println("\t logURL");
pw.print(" container_1234_0005_01_000002");
pw.print("\t"+dateFormat.format(new Date(time1)));
pw.print("\t"+dateFormat.format(new Date(time2)));
pw.print("\t COMPLETE");
pw.print("\t host:1234");
pw.print("\t http://host:2345");
pw.println("\t logURL");
pw.print(" container_1234_0005_01_000003");
pw.print("\t"+dateFormat.format(new Date(time1)));
pw.print("\t N/A");
pw.print("\t RUNNING");
pw.print("\t host:1234");
pw.print("\t http://host:2345");
pw.println("\t ");
pw.close();
String appReportStr = baos.toString("UTF-8");

View File

@ -338,4 +338,23 @@ private ContainerState convertFromProtoFormat(
ContainerStateProto containerState) {
return ProtoUtils.convertFromProtoFormat(containerState);
}
@Override
public String getNodeHttpAddress() {
ContainerReportProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasNodeHttpAddress()) {
return null;
}
return (p.getNodeHttpAddress());
}
@Override
public void setNodeHttpAddress(String nodeHttpAddress) {
maybeInitBuilder();
if (nodeHttpAddress == null) {
builder.clearNodeHttpAddress();
return;
}
builder.setNodeHttpAddress(nodeHttpAddress);
}
}

View File

@ -215,7 +215,7 @@ private ContainerReport convertToContainerReport(
containerHistory.getStartTime(), containerHistory.getFinishTime(),
containerHistory.getDiagnosticsInfo(), logUrl,
containerHistory.getContainerExitStatus(),
containerHistory.getContainerState());
containerHistory.getContainerState(), null);
}
@Override

View File

@ -415,6 +415,7 @@ private static ContainerReport convertToContainerReport(
String diagnosticsInfo = null;
int exitStatus = ContainerExitStatus.INVALID;
ContainerState state = null;
String nodeHttpAddress = null;
Map<String, Object> entityInfo = entity.getOtherInfo();
if (entityInfo != null) {
if (entityInfo
@ -444,6 +445,12 @@ private static ContainerReport convertToContainerReport(
allocatedPriority = (Integer) entityInfo.get(
ContainerMetricsConstants.ALLOCATED_PRIORITY_ENTITY_INFO);
}
if (entityInfo.containsKey(
ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO)) {
nodeHttpAddress =
(String) entityInfo
.get(ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO);
}
}
List<TimelineEvent> events = entity.getEvents();
if (events != null) {
@ -493,7 +500,8 @@ private static ContainerReport convertToContainerReport(
Resource.newInstance(allocatedMem, allocatedVcore),
NodeId.newInstance(allocatedHost, allocatedPort),
Priority.newInstance(allocatedPriority),
createdTime, finishedTime, diagnosticsInfo, logUrl, exitStatus, state);
createdTime, finishedTime, diagnosticsInfo, logUrl, exitStatus, state,
nodeHttpAddress);
}
private ApplicationReportExt generateApplicationReport(TimelineEntity entity,

View File

@ -57,4 +57,6 @@ public class ContainerMetricsConstants {
public static final String STATE_EVENT_INFO =
"YARN_CONTAINER_STATE";
public static final String ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO =
"YARN_CONTAINER_ALLOCATED_HOST_HTTP_ADDRESS";
}

View File

@ -199,12 +199,14 @@ public Collection<ContainerReport> run() throws Exception {
.append(url("container", container.getContainerId()))
.append("'>")
.append(container.getContainerId())
.append("</a>\",\"<a href='")
.append("#") // TODO: replace with node http address (YARN-1884)
.append("</a>\",\"<a ")
.append(
container.getNodeHttpAddress() == null ? "#" : "href='"
+ container.getNodeHttpAddress())
.append("'>")
.append(container.getAssignedNodeId() == null ? "N/A" :
.append(container.getNodeHttpAddress() == null ? "N/A" :
StringEscapeUtils.escapeJavaScript(StringEscapeUtils
.escapeHtml(container.getAssignedNodeId())))
.escapeHtml(container.getNodeHttpAddress())))
.append("</a>\",\"")
.append(container.getContainerExitStatus()).append("\",\"<a href='")
.append(container.getLogUrl() == null ?
@ -271,4 +273,4 @@ private boolean hasAMContainer(ContainerId containerId,
}
return false;
}
}
}

View File

@ -285,15 +285,12 @@ public ContainerReport run() throws Exception {
}
long startTime = 0L;
String logsLink = null;
String nodeLink = null;
if (containerReport != null) {
ContainerInfo container = new ContainerInfo(containerReport);
startTime = container.getStartedTime();
logsLink = containerReport.getLogUrl();
}
String nodeLink = null;
if (appAttempt.getHost() != null && appAttempt.getRpcPort() >= 0
&& appAttempt.getRpcPort() < 65536) {
nodeLink = appAttempt.getHost() + ":" + appAttempt.getRpcPort();
nodeLink = containerReport.getNodeHttpAddress();
}
// AppAttemptID numerical value parsed by parseHadoopID in
// yarn.dt.plugins.js
@ -304,8 +301,8 @@ public ContainerReport run() throws Exception {
.append(appAttempt.getAppAttemptId())
.append("</a>\",\"")
.append(startTime)
.append("\",\"<a href='")
.append("#") // TODO: replace with node http address (YARN-1884)
.append("\",\"<a ")
.append(nodeLink == null ? "#" : "href='" + nodeLink)
.append("'>")
.append(nodeLink == null ? "N/A" : StringEscapeUtils
.escapeJavaScript(StringEscapeUtils.escapeHtml(nodeLink)))

View File

@ -104,7 +104,12 @@ public ContainerReport run() throws Exception {
container.getContainerState() == null ? UNAVAILABLE : container
.getContainerState())
._("Exit Status:", container.getContainerExitStatus())
._("Node:", container.getAssignedNodeId())
._(
"Node:",
container.getNodeHttpAddress() == null ? "#" : container
.getNodeHttpAddress(),
container.getNodeHttpAddress() == null ? "N/A" : container
.getNodeHttpAddress())
._("Priority:", container.getPriority())
._("Started:", Times.format(container.getStartedTime()))
._(

View File

@ -42,6 +42,7 @@ public class ContainerInfo {
protected String logUrl;
protected int containerExitStatus;
protected ContainerState containerState;
protected String nodeHttpAddress;
public ContainerInfo() {
// JAXB needs this
@ -64,6 +65,7 @@ public ContainerInfo(ContainerReport container) {
logUrl = container.getLogUrl();
containerExitStatus = container.getContainerExitStatus();
containerState = container.getContainerState();
nodeHttpAddress = container.getNodeHttpAddress();
}
public String getContainerId() {
@ -114,4 +116,7 @@ public ContainerState getContainerState() {
return containerState;
}
public String getNodeHttpAddress() {
return nodeHttpAddress;
}
}

View File

@ -29,18 +29,21 @@ public class ContainerCreatedEvent extends SystemMetricsEvent {
private Resource allocatedResource;
private NodeId allocatedNode;
private Priority allocatedPriority;
private String nodeHttpAddress;
public ContainerCreatedEvent(
ContainerId containerId,
Resource allocatedResource,
NodeId allocatedNode,
Priority allocatedPriority,
long createdTime) {
long createdTime,
String nodeHttpAddress) {
super(SystemMetricsEventType.CONTAINER_CREATED, createdTime);
this.containerId = containerId;
this.allocatedResource = allocatedResource;
this.allocatedNode = allocatedNode;
this.allocatedPriority = allocatedPriority;
this.nodeHttpAddress = nodeHttpAddress;
}
@Override
@ -64,4 +67,7 @@ public Priority getAllocatedPriority() {
return allocatedPriority;
}
public String getNodeHttpAddress() {
return nodeHttpAddress;
}
}

View File

@ -181,7 +181,7 @@ public void containerCreated(RMContainer container, long createdTime) {
container.getAllocatedResource(),
container.getAllocatedNode(),
container.getAllocatedPriority(),
createdTime));
createdTime, container.getNodeHttpAddress()));
}
}
@ -388,6 +388,9 @@ private void publishContainerCreatedEvent(ContainerCreatedEvent event) {
event.getAllocatedNode().getPort());
entityInfo.put(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENTITY_INFO,
event.getAllocatedPriority().getPriority());
entityInfo.put(
ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO,
event.getNodeHttpAddress());
entity.setOtherInfo(entityInfo);
TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType(ContainerMetricsConstants.CREATED_EVENT_TYPE);

View File

@ -79,4 +79,5 @@ public interface RMContainer extends EventHandler<RMContainerEvent> {
List<ResourceRequest> getResourceRequests();
String getNodeHttpAddress();
}

View File

@ -41,7 +41,6 @@
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRunningOnNodeEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerAllocatedEvent;
@ -573,10 +572,29 @@ public ContainerReport createContainerReport() {
this.getAllocatedResource(), this.getAllocatedNode(),
this.getAllocatedPriority(), this.getCreationTime(),
this.getFinishTime(), this.getDiagnosticsInfo(), this.getLogURL(),
this.getContainerExitStatus(), this.getContainerState());
this.getContainerExitStatus(), this.getContainerState(),
this.getNodeHttpAddress());
} finally {
this.readLock.unlock();
}
return containerReport;
}
@Override
public String getNodeHttpAddress() {
try {
readLock.lock();
if (container.getNodeHttpAddress() != null) {
StringBuilder httpAddress = new StringBuilder();
httpAddress.append(WebAppUtils.getHttpSchemePrefix(rmContext
.getYarnConfiguration()));
httpAddress.append(container.getNodeHttpAddress());
return httpAddress.toString();
} else {
return null;
}
} finally {
readLock.unlock();
}
}
}

View File

@ -38,7 +38,6 @@
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer;
import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp;
import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
@ -386,6 +385,10 @@ private static RMContainer createRMContainer(ContainerId containerId) {
when(container.getDiagnosticsInfo()).thenReturn("test diagnostics info");
when(container.getContainerExitStatus()).thenReturn(-1);
when(container.getContainerState()).thenReturn(ContainerState.COMPLETE);
Container mockContainer = mock(Container.class);
when(container.getContainer()).thenReturn(mockContainer);
when(mockContainer.getNodeHttpAddress())
.thenReturn("http://localhost:1234");
return container;
}