big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-1685. Fixed few bugs related to handling of containers' log-URLs on ResourceManager and history-service. Contributed by Zhijie Shen.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1578602 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2014-03-17 21:36:21 +00:00
parent 6318afe3b9
commit 8314674947
22 changed files with 107 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -495,6 +495,9 @@ Release 2.4.0 - UNRELEASED
YARN-1830. Fixed TestRMRestart#testQueueMetricsOnRMRestart failure due to
race condition when app is submitted. (Zhijie Shen via jianhe)
YARN-1685. Fixed few bugs related to handling of containers' log-URLs on
ResourceManager and history-service. (Zhijie Shen via vinodkv)
Release 2.3.1 - UNRELEASED
INCOMPATIBLE CHANGES

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto
View File

@ -90,9 +90,8 @@ message ContainerHistoryDataProto {
optional int64 start_time = 5;
optional int64 finish_time = 6;
optional string diagnostics_info = 7;
optional string log_url = 8;
optional int32 container_exit_status = 9;
optional ContainerStateProto container_state = 10;
optional int32 container_exit_status = 8;
optional ContainerStateProto container_state = 9;
}
message ContainerStartDataProto {
@ -107,7 +106,6 @@ message ContainerFinishDataProto {
optional ContainerIdProto container_id = 1;
optional int64 finish_time = 2;
optional string diagnostics_info = 3;
optional string log_url = 4;
optional int32 container_exit_status = 5;
optional ContainerStateProto container_state = 6;
optional int32 container_exit_status = 4;
optional ContainerStateProto container_state = 5;
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java
View File

@ -178,8 +178,5 @@ private static void uappend(StringBuilder sb, String part) {
public static String percent(double value) {
return String.format("%.2f", value * 100);
}
public static String getPartUrl(String url, String part) {
return url.substring(url.indexOf(part));
}
}

33
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
View File

@ -17,7 +17,7 @@
*/
package org.apache.hadoop.yarn.webapp.util;
import static org.apache.hadoop.yarn.util.StringHelper.join;
import static org.apache.hadoop.yarn.util.StringHelper.PATH_JOINER;
import java.net.InetAddress;
import java.net.InetSocketAddress;
@ -29,9 +29,7 @@
import org.apache.hadoop.http.HttpConfig.Policy;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.ConverterUtils;
@Private
@Evolving
@ -169,18 +167,35 @@ public static String getURLWithScheme(String schemePrefix, String url) {
}
}
public static String getLogUrl(String nodeHttpAddress, String allocatedNode,
ContainerId containerId, String user) {
return join("//", nodeHttpAddress, "/logs", "/",
allocatedNode, "/", ConverterUtils.toString(containerId), "/",
ConverterUtils.toString(containerId), "/", user);
public static String getRunningLogURL(
String nodeHttpAddress, String containerId, String user) {
if (nodeHttpAddress == null || nodeHttpAddress.isEmpty() ||
containerId == null || containerId.isEmpty() ||
user == null || user.isEmpty()) {
return null;
}
return PATH_JOINER.join(
nodeHttpAddress, "node", "containerlogs", containerId, user);
}
public static String getAggregatedLogURL(String serverHttpAddress,
String allocatedNode, String containerId, String entity, String user) {
if (serverHttpAddress == null || serverHttpAddress.isEmpty() ||
allocatedNode == null || allocatedNode.isEmpty() ||
containerId == null || containerId.isEmpty() ||
entity == null || entity.isEmpty() ||
user == null || user.isEmpty()) {
return null;
}
return PATH_JOINER.join(serverHttpAddress, "applicationhistory", "logs",
allocatedNode, containerId, entity, user);
}
/**
* Choose which scheme (HTTP or HTTPS) to use when generating a URL based on
* the configuration.
*
* @return the schmeme (HTTP / HTTPS)
* @return the scheme (HTTP / HTTPS)
*/
public static String getHttpSchemePrefix(Configuration conf) {
return YarnConfiguration.useHttps(conf) ? HTTPS_PREFIX : HTTP_PREFIX;

30
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
View File

@ -39,6 +39,7 @@
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import com.google.common.annotations.VisibleForTesting;
@ -49,6 +50,7 @@ public class ApplicationHistoryManagerImpl extends AbstractService implements
private static final String UNAVAILABLE = "N/A";
private ApplicationHistoryStore historyStore;
private String serverHttpAddress;
public ApplicationHistoryManagerImpl() {
super(ApplicationHistoryManagerImpl.class.getName());
@ -59,6 +61,8 @@ protected void serviceInit(Configuration conf) throws Exception {
LOG.info("ApplicationHistory Init");
historyStore = createApplicationHistoryStore(conf);
historyStore.init(conf);
serverHttpAddress = WebAppUtils.getHttpSchemePrefix(conf) +
WebAppUtils.getAHSWebAppURLWithoutScheme(conf);
super.serviceInit(conf);
}
@ -87,7 +91,10 @@ protected ApplicationHistoryStore createApplicationHistoryStore(
@Override
public ContainerReport getAMContainer(ApplicationAttemptId appAttemptId)
throws IOException {
return convertToContainerReport(historyStore.getAMContainer(appAttemptId));
ApplicationReport app =
getApplication(appAttemptId.getApplicationId());
return convertToContainerReport(historyStore.getAMContainer(appAttemptId),
app == null ? null : app.getUser());
}
@Override
@ -187,16 +194,26 @@ public ApplicationAttemptReport getApplicationAttempt(
@Override
public ContainerReport getContainer(ContainerId containerId)
throws IOException {
return convertToContainerReport(historyStore.getContainer(containerId));
ApplicationReport app =
getApplication(containerId.getApplicationAttemptId().getApplicationId());
return convertToContainerReport(historyStore.getContainer(containerId),
app == null ? null: app.getUser());
}
private ContainerReport convertToContainerReport(
ContainerHistoryData containerHistory) {
ContainerHistoryData containerHistory, String user) {
// If the container has the aggregated log, add the server root url
String logUrl = WebAppUtils.getAggregatedLogURL(
serverHttpAddress,
containerHistory.getAssignedNode().toString(),
containerHistory.getContainerId().toString(),
containerHistory.getContainerId().toString(),
user);
return ContainerReport.newInstance(containerHistory.getContainerId(),
containerHistory.getAllocatedResource(),
containerHistory.getAssignedNode(), containerHistory.getPriority(),
containerHistory.getStartTime(), containerHistory.getFinishTime(),
containerHistory.getDiagnosticsInfo(), containerHistory.getLogURL(),
containerHistory.getDiagnosticsInfo(), logUrl,
containerHistory.getContainerExitStatus(),
containerHistory.getContainerState());
}
@ -204,13 +221,16 @@ private ContainerReport convertToContainerReport(
@Override
public Map<ContainerId, ContainerReport> getContainers(
ApplicationAttemptId appAttemptId) throws IOException {
ApplicationReport app =
getApplication(appAttemptId.getApplicationId());
Map<ContainerId, ContainerHistoryData> histData =
historyStore.getContainers(appAttemptId);
HashMap<ContainerId, ContainerReport> containersReport =
new HashMap<ContainerId, ContainerReport>();
for (Entry<ContainerId, ContainerHistoryData> entry : histData.entrySet()) {
containersReport.put(entry.getKey(),
convertToContainerReport(entry.getValue()));
convertToContainerReport(entry.getValue(),
app == null ? null : app.getUser()));
}
return containersReport;
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
View File

@ -315,7 +315,7 @@ public ContainerHistoryData getContainer(ContainerId containerId)
ContainerHistoryData historyData =
ContainerHistoryData
.newInstance(containerId, null, null, null, Long.MIN_VALUE,
Long.MAX_VALUE, null, null, Integer.MAX_VALUE, null);
Long.MAX_VALUE, null, Integer.MAX_VALUE, null);
while ((!readStartData || !readFinishData) && hfReader.hasNext()) {
HistoryFileReader.Entry entry = hfReader.next();
if (entry.key.id.equals(containerId.toString())) {
@ -382,7 +382,7 @@ public Map<ContainerId, ContainerHistoryData> getContainers(
if (historyData == null) {
historyData = ContainerHistoryData.newInstance(
containerId, null, null, null, Long.MIN_VALUE,
Long.MAX_VALUE, null, null, Integer.MAX_VALUE, null);
Long.MAX_VALUE, null, Integer.MAX_VALUE, null);
historyDataMap.put(containerId, historyData);
}
if (entry.key.suffix.equals(START_DATA_SUFFIX)) {
@ -632,7 +632,6 @@ private static void mergeContainerHistoryData(
ContainerHistoryData historyData, ContainerFinishData finishData) {
historyData.setFinishTime(finishData.getFinishTime());
historyData.setDiagnosticsInfo(finishData.getDiagnosticsInfo());
historyData.setLogURL(finishData.getLogURL());
historyData.setContainerExitStatus(finishData.getContainerExitStatus());
historyData.setContainerState(finishData.getContainerState());
}

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java
View File

@ -233,7 +233,7 @@ public void containerStarted(ContainerStartData containerStart)
ContainerHistoryData.newInstance(containerStart.getContainerId(),
containerStart.getAllocatedResource(),
containerStart.getAssignedNode(), containerStart.getPriority(),
containerStart.getStartTime(), Long.MAX_VALUE, null, null,
containerStart.getStartTime(), Long.MAX_VALUE, null,
Integer.MAX_VALUE, null));
if (oldData != null) {
throw new IOException("The start information of container "
@ -260,7 +260,6 @@ public void containerFinished(ContainerFinishData containerFinish)
}
data.setFinishTime(containerFinish.getFinishTime());
data.setDiagnosticsInfo(containerFinish.getDiagnosticsInfo());
data.setLogURL(containerFinish.getLogURL());
data.setContainerExitStatus(containerFinish.getContainerExitStatus());
data.setContainerState(containerFinish.getContainerState());
}

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerFinishData.java
View File

@ -35,14 +35,13 @@ public abstract class ContainerFinishData {
@Public
@Unstable
public static ContainerFinishData newInstance(ContainerId containerId,
long finishTime, String diagnosticsInfo, String logURL,
int containerExitCode, ContainerState containerState) {
long finishTime, String diagnosticsInfo, int containerExitCode,
ContainerState containerState) {
ContainerFinishData containerFD =
Records.newRecord(ContainerFinishData.class);
containerFD.setContainerId(containerId);
containerFD.setFinishTime(finishTime);
containerFD.setDiagnosticsInfo(diagnosticsInfo);
containerFD.setLogURL(logURL);
containerFD.setContainerExitStatus(containerExitCode);
containerFD.setContainerState(containerState);
return containerFD;
@ -72,14 +71,6 @@ public static ContainerFinishData newInstance(ContainerId containerId,
@Unstable
public abstract void setDiagnosticsInfo(String diagnosticsInfo);
@Public
@Unstable
public abstract String getLogURL();
@Public
@Unstable
public abstract void setLogURL(String logURL);
@Public
@Unstable
public abstract int getContainerExitStatus();

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerHistoryData.java
View File

@ -48,8 +48,6 @@ public class ContainerHistoryData {
private String diagnosticsInfo;
private String logURL;
private int containerExitStatus;
private ContainerState containerState;
@ -58,7 +56,7 @@ public class ContainerHistoryData {
@Unstable
public static ContainerHistoryData newInstance(ContainerId containerId,
Resource allocatedResource, NodeId assignedNode, Priority priority,
long startTime, long finishTime, String diagnosticsInfo, String logURL,
long startTime, long finishTime, String diagnosticsInfo,
int containerExitCode, ContainerState containerState) {
ContainerHistoryData containerHD = new ContainerHistoryData();
containerHD.setContainerId(containerId);
@ -68,7 +66,6 @@ public static ContainerHistoryData newInstance(ContainerId containerId,
containerHD.setStartTime(startTime);
containerHD.setFinishTime(finishTime);
containerHD.setDiagnosticsInfo(diagnosticsInfo);
containerHD.setLogURL(logURL);
containerHD.setContainerExitStatus(containerExitCode);
containerHD.setContainerState(containerState);
return containerHD;
@ -158,18 +155,6 @@ public void setDiagnosticsInfo(String diagnosticsInfo) {
this.diagnosticsInfo = diagnosticsInfo;
}
@Public
@Unstable
public String getLogURL() {
return logURL;
}
@Public
@Unstable
public void setLogURL(String logURL) {
this.logURL = logURL;
}
@Public
@Unstable
public int getContainerExitStatus() {

19
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerFinishDataPBImpl.java
View File

@ -101,25 +101,6 @@ public void setDiagnosticsInfo(String diagnosticsInfo) {
builder.setDiagnosticsInfo(diagnosticsInfo);
}
@Override
public String getLogURL() {
ContainerFinishDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasLogUrl()) {
return null;
}
return p.getLogUrl();
}
@Override
public void setLogURL(String logURL) {
maybeInitBuilder();
if (logURL == null) {
builder.clearLogUrl();
return;
}
builder.setLogUrl(logURL);
}
@Override
public int getContainerExitStatus() {
ContainerFinishDataProtoOrBuilder p = viaProto ? proto : builder;

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java
View File

@ -78,8 +78,7 @@ protected void writeContainerStartData(ContainerId containerId)
protected void writeContainerFinishData(ContainerId containerId)
throws IOException {
store.containerFinished(ContainerFinishData.newInstance(containerId, 0,
containerId.toString(), "http://localhost:0/log", 0,
ContainerState.COMPLETE));
containerId.toString(), 0, ContainerState.COMPLETE));
}
}

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
View File

@ -44,6 +44,7 @@
import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@ -52,11 +53,16 @@ public class TestApplicationHistoryClientService extends
ApplicationHistoryStoreTestUtils {
ApplicationHistoryServer historyServer = null;
String expectedLogUrl = null;
@Before
public void setup() {
historyServer = new ApplicationHistoryServer();
Configuration config = new YarnConfiguration();
expectedLogUrl = WebAppUtils.getHttpSchemePrefix(config) +
WebAppUtils.getAHSWebAppURLWithoutScheme(config) +
"/applicationhistory/logs/localhost:0/container_0_0001_01_000001/" +
"container_0_0001_01_000001/test user";
config.setClass(YarnConfiguration.APPLICATION_HISTORY_STORE,
MemoryApplicationHistoryStore.class, ApplicationHistoryStore.class);
historyServer.init(config);
@ -156,11 +162,13 @@ public void testApplicationAttempts() throws IOException, YarnException {
@Test
public void testContainerReport() throws IOException, YarnException {
ApplicationId appId = ApplicationId.newInstance(0, 1);
writeApplicationStartData(appId);
ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, 1);
ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);
writeContainerStartData(containerId);
writeContainerFinishData(containerId);
writeApplicationFinishData(appId);
GetContainerReportRequest request =
GetContainerReportRequest.newInstance(containerId);
GetContainerReportResponse response =
@ -169,11 +177,13 @@ public void testContainerReport() throws IOException, YarnException {
ContainerReport container = response.getContainerReport();
Assert.assertNotNull(container);
Assert.assertEquals(containerId, container.getContainerId());
Assert.assertEquals(expectedLogUrl, container.getLogUrl());
}
@Test
public void testContainers() throws IOException, YarnException {
ApplicationId appId = ApplicationId.newInstance(0, 1);
writeApplicationStartData(appId);
ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, 1);
ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);
@ -182,6 +192,7 @@ public void testContainers() throws IOException, YarnException {
writeContainerFinishData(containerId);
writeContainerStartData(containerId1);
writeContainerFinishData(containerId1);
writeApplicationFinishData(appId);
GetContainersRequest request =
GetContainersRequest.newInstance(appAttemptId);
GetContainersResponse response =

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
View File

@ -25,6 +25,7 @@
import org.junit.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
@ -34,12 +35,14 @@
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.api.ApplicationContext;
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryManager;
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryStore;
import org.apache.hadoop.yarn.server.applicationhistoryservice.MemoryApplicationHistoryStore;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
@ -287,7 +290,12 @@ public void testSingleContainer() throws Exception {
container.getString("assignedNodeId"));
assertEquals(Priority.newInstance(containerId.getId()).toString(),
container.getString("priority"));
assertEquals("http://localhost:0/log", container.getString("logUrl"));
Configuration conf = new YarnConfiguration();
assertEquals(WebAppUtils.getHttpSchemePrefix(conf) +
WebAppUtils.getAHSWebAppURLWithoutScheme(conf) +
"/applicationhistory/logs/localhost:0/container_0_0001_01_000001/" +
"container_0_0001_01_000001/test user",
container.getString("logUrl"));
assertEquals(ContainerState.COMPLETE.toString(),
container.getString("containerState"));
}

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
View File

@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.server.webapp;
import static org.apache.hadoop.yarn.util.StringHelper.join;
import static org.apache.hadoop.yarn.util.StringHelper.getPartUrl;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPLICATION_ATTEMPT_ID;
import java.io.IOException;
@ -127,8 +126,6 @@ protected void render(Block html) {
StringBuilder containersTableData = new StringBuilder("[\n");
for (ContainerReport containerReport : containers) {
String logURL = containerReport.getLogUrl();
logURL = getPartUrl(logURL, "log");
ContainerInfo container = new ContainerInfo(containerReport);
// ConatinerID numerical value parsed by parseHadoopID in
// yarn.dt.plugins.js
@ -144,8 +141,10 @@ protected void render(Block html) {
StringEscapeUtils.escapeJavaScript(StringEscapeUtils
.escapeHtml(container.getAssignedNodeId()))).append("</a>\",\"")
.append(container.getContainerExitStatus()).append("\",\"<a href='")
.append(logURL == null ? "#" : url(logURL)).append("'>")
.append(logURL == null ? "N/A" : "Logs").append("</a>\"],\n");
.append(container.getLogUrl() == null ?
"#" : container.getLogUrl()).append("'>")
.append(container.getLogUrl() == null ?
"N/A" : "Logs").append("</a>\"],\n");
}
if (containersTableData.charAt(containersTableData.length() - 2) == ',') {
containersTableData.delete(containersTableData.length() - 2,

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
View File

@ -19,14 +19,12 @@
package org.apache.hadoop.yarn.server.webapp;
import static org.apache.hadoop.yarn.util.StringHelper.join;
import static org.apache.hadoop.yarn.util.StringHelper.getPartUrl;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPLICATION_ID;
import java.io.IOException;
import java.util.Collection;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@ -145,7 +143,6 @@ protected void render(Block html) {
ContainerInfo container = new ContainerInfo(containerReport);
startTime = container.getStartedTime();
logsLink = containerReport.getLogUrl();
logsLink = getPartUrl(logsLink, "log");
}
String nodeLink = null;
if (appAttempt.getHost() != null && appAttempt.getRpcPort() >= 0
@ -169,8 +166,8 @@ protected void render(Block html) {
nodeLink == null ? "N/A" : StringEscapeUtils
.escapeJavaScript(StringEscapeUtils.escapeHtml(nodeLink)))
.append("</a>\",\"<a href='")
.append(logsLink == null ? "#" : url(logsLink)).append("'>")
.append(nodeLink == null ? "N/A" : "Logs").append("</a>\"],\n");
.append(logsLink == null ? "#" : logsLink).append("'>")
.append(logsLink == null ? "N/A" : "Logs").append("</a>\"],\n");
}
if (attemptsTableData.charAt(attemptsTableData.length() - 2) == ',') {
attemptsTableData.delete(attemptsTableData.length() - 2,

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java
View File

@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.server.webapp;
import static org.apache.hadoop.yarn.util.StringHelper.join;
import static org.apache.hadoop.yarn.util.StringHelper.getPartUrl;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.CONTAINER_ID;
import java.io.IOException;
@ -79,8 +78,6 @@ protected void render(Block html) {
}
ContainerInfo container = new ContainerInfo(containerReport);
String logURL = containerReport.getLogUrl();
logURL = getPartUrl(logURL, "log");
setTitle(join("Container ", containerid));
info("Container Overview")
@ -97,7 +94,8 @@ protected void render(Block html) {
"Resource:",
container.getAllocatedMB() + " Memory, "
+ container.getAllocatedVCores() + " VCores")
._("Logs:", logURL == null ? "#" : url(logURL), "Logs")
._("Logs:", container.getLogUrl() == null ? "#" : container.getLogUrl(),
container.getLogUrl() == null ? "N/A" : "Logs")
._("Diagnostics:", container.getDiagnosticsInfo());
html._(InfoBlock.class);

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java
View File

@ -280,7 +280,7 @@ public void containerFinished(RMContainer container) {
new WritingContainerFinishEvent(container.getContainerId(),
ContainerFinishData.newInstance(container.getContainerId(),
container.getFinishTime(), container.getDiagnosticsInfo(),
container.getLogURL(), container.getContainerExitStatus(),
container.getContainerExitStatus(),
container.getContainerState())));
}
}

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
View File

@ -44,6 +44,7 @@
import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
@SuppressWarnings({"unchecked", "rawtypes"})
@ -148,7 +149,6 @@ RMContainerEventType.RELEASED, new KillTransition())
private Priority reservedPriority;
private long startTime;
private long finishTime;
private String logURL;
private ContainerStatus finishedStatus;
@ -264,7 +264,8 @@ public String getDiagnosticsInfo() {
public String getLogURL() {
try {
readLock.lock();
return logURL;
return WebAppUtils.getRunningLogURL("//" + container.getNodeHttpAddress(),
ConverterUtils.toString(containerId), user);
} finally {
readLock.unlock();
}
@ -380,11 +381,6 @@ private static final class LaunchedTransition extends BaseTransition {
@Override
public void transition(RMContainerImpl container, RMContainerEvent event) {
// The logs of running containers should be found on NM webUI
// The logs should be accessible after the container is launched
container.logURL = WebAppUtils.getLogUrl(container.container
.getNodeHttpAddress(), container.getAllocatedNode().toString(),
container.containerId, container.user);
// Unregister from containerAllocationExpirer.
container.containerAllocationExpirer.unregister(container
.getContainerId());
@ -399,9 +395,6 @@ public void transition(RMContainerImpl container, RMContainerEvent event) {
container.finishTime = System.currentTimeMillis();
container.finishedStatus = finishedEvent.getRemoteContainerStatus();
// TODO: when AHS webUI is ready, logURL should be updated to point to
// the web page that will show the aggregated logs
// Inform AppAttempt
container.eventHandler.handle(new RMAppAttemptContainerFinishedEvent(
container.appAttemptId, finishedEvent.getRemoteContainerStatus()));
@ -415,7 +408,6 @@ private static final class ContainerFinishedAtAcquiredState extends
FinishedTransition {
@Override
public void transition(RMContainerImpl container, RMContainerEvent event) {
// Unregister from containerAllocationExpirer.
container.containerAllocationExpirer.unregister(container
.getContainerId());

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java
View File

@ -17,8 +17,6 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import static org.apache.hadoop.yarn.util.StringHelper.join;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
@ -26,6 +24,7 @@
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
@XmlRootElement(name = "appAttempt")
@XmlAccessorType(XmlAccessType.FIELD)
@ -55,10 +54,9 @@ public AppAttemptInfo(RMAppAttempt attempt, String user) {
this.containerId = masterContainer.getId().toString();
this.nodeHttpAddress = masterContainer.getNodeHttpAddress();
this.nodeId = masterContainer.getNodeId().toString();
this.logsLink = join("//",
masterContainer.getNodeHttpAddress(),
"/node", "/containerlogs/",
ConverterUtils.toString(masterContainer.getId()), "/", user);
this.logsLink =
WebAppUtils.getRunningLogURL("//" + masterContainer.getNodeHttpAddress(),
ConverterUtils.toString(masterContainer.getId()), user);
}
}
}

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
View File

@ -17,14 +17,11 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import static org.apache.hadoop.yarn.util.StringHelper.join;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlTransient;
import com.google.common.base.Joiner;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.Container;
@ -38,6 +35,8 @@
import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import com.google.common.base.Joiner;
@XmlRootElement(name = "app")
@XmlAccessorType(XmlAccessType.FIELD)
public class AppInfo {
@ -131,12 +130,10 @@ public AppInfo(RMApp app, Boolean hasAccess, String schemePrefix) {
Container masterContainer = attempt.getMasterContainer();
if (masterContainer != null) {
this.amContainerLogsExist = true;
String url = join(schemePrefix,
masterContainer.getNodeHttpAddress(),
"/node", "/containerlogs/",
this.amContainerLogs = WebAppUtils.getRunningLogURL(
schemePrefix + masterContainer.getNodeHttpAddress(),
ConverterUtils.toString(masterContainer.getId()),
"/", app.getUser());
this.amContainerLogs = url;
app.getUser());
this.amHostHttpAddress = masterContainer.getNodeHttpAddress();
}

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java
View File

@ -296,7 +296,6 @@ public void testWriteContainer() throws Exception {
}
Assert.assertEquals("test diagnostics info",
containerHD.getDiagnosticsInfo());
Assert.assertEquals("test log url", containerHD.getLogURL());
Assert.assertEquals(-1, containerHD.getContainerExitStatus());
Assert.assertEquals(ContainerState.COMPLETE,
containerHD.getContainerState());

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
View File

@ -104,8 +104,7 @@ public void testReleaseWhileRunning() {
RMContainerEventType.LAUNCHED));
drainDispatcher.await();
assertEquals(RMContainerState.RUNNING, rmContainer.getState());
assertEquals(
"//host:3465/logs/host:3425/container_1_0001_01_000001/container_1_0001_01_000001/user",
assertEquals("//host:3465/node/containerlogs/container_1_0001_01_000001/user",
rmContainer.getLogURL());
// In RUNNING state. Verify RELEASED and associated actions.
@ -191,8 +190,7 @@ public void testExpireWhileRunning() {
RMContainerEventType.LAUNCHED));
drainDispatcher.await();
assertEquals(RMContainerState.RUNNING, rmContainer.getState());
assertEquals(
"//host:3465/logs/host:3425/container_1_0001_01_000001/container_1_0001_01_000001/user",
assertEquals("//host:3465/node/containerlogs/container_1_0001_01_000001/user",
rmContainer.getLogURL());
// In RUNNING state. Verify EXPIRE and associated actions.