YARN-10083. Provide utility to ask whether an application is in final status. Contributed by Adam Antal

This commit is contained in:
Szilard Nemeth 2020-01-22 16:04:35 +01:00
parent bd03053ea2
commit 9520b2ad79
16 changed files with 85 additions and 92 deletions

View File

@ -69,6 +69,7 @@
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.util.Apps;
public class NotRunningJob implements MRClientProtocol { public class NotRunningJob implements MRClientProtocol {
@ -138,9 +139,7 @@ public GetJobReportResponse getJobReport(GetJobReportRequest request)
jobReport.setUser(applicationReport.getUser()); jobReport.setUser(applicationReport.getUser());
jobReport.setStartTime(applicationReport.getStartTime()); jobReport.setStartTime(applicationReport.getStartTime());
YarnApplicationState state = applicationReport.getYarnApplicationState(); YarnApplicationState state = applicationReport.getYarnApplicationState();
if (state == YarnApplicationState.KILLED if (Apps.isApplicationFinalState(state)) {
|| state == YarnApplicationState.FAILED
|| state == YarnApplicationState.FINISHED) {
jobReport.setDiagnostics(applicationReport.getDiagnostics()); jobReport.setDiagnostics(applicationReport.getDiagnostics());
} }
jobReport.setJobName(applicationReport.getName()); jobReport.setJobName(applicationReport.getName());

View File

@ -94,6 +94,7 @@
import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenSelector; import org.apache.hadoop.yarn.security.client.RMDelegationTokenSelector;
import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.UnitsConversionUtil; import org.apache.hadoop.yarn.util.UnitsConversionUtil;
import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.ResourceUtils;
@ -899,9 +900,7 @@ private void killUnFinishedApplication(ApplicationId appId)
} catch (YarnException e) { } catch (YarnException e) {
throw new IOException(e); throw new IOException(e);
} }
if (application.getYarnApplicationState() == YarnApplicationState.FINISHED if (Apps.isApplicationFinalState(application.getYarnApplicationState())) {
|| application.getYarnApplicationState() == YarnApplicationState.FAILED
|| application.getYarnApplicationState() == YarnApplicationState.KILLED) {
return; return;
} }
killApplication(appId); killApplication(appId);

View File

@ -96,6 +96,7 @@
import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.hadoop.yarn.client.api.YarnClientApplication;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.Records;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -891,7 +892,8 @@ private boolean monitorInfraApplication() throws YarnException, IOException {
boolean success = false; boolean success = false;
Thread namenodeMonitoringThread = new Thread(() -> { Thread namenodeMonitoringThread = new Thread(() -> {
Supplier<Boolean> exitCritera = () -> isCompleted(infraAppState); Supplier<Boolean> exitCritera = () ->
Apps.isApplicationFinalState(infraAppState);
Optional<Properties> namenodeProperties = Optional.empty(); Optional<Properties> namenodeProperties = Optional.empty();
while (!exitCritera.get()) { while (!exitCritera.get()) {
try { try {
@ -925,7 +927,7 @@ private boolean monitorInfraApplication() throws YarnException, IOException {
return; return;
} }
} }
if (!isCompleted(infraAppState) && launchWorkloadJob) { if (!Apps.isApplicationFinalState(infraAppState) && launchWorkloadJob) {
launchAndMonitorWorkloadDriver(namenodeProperties.get()); launchAndMonitorWorkloadDriver(namenodeProperties.get());
} }
}); });
@ -1050,7 +1052,8 @@ private void launchAndMonitorWorkloadDriver(Properties nameNodeProperties) {
workloadJob = WorkloadDriver.getJobForSubmission(workloadConf, workloadJob = WorkloadDriver.getJobForSubmission(workloadConf,
nameNodeURI.toString(), workloadStartTime, AuditReplayMapper.class); nameNodeURI.toString(), workloadStartTime, AuditReplayMapper.class);
workloadJob.submit(); workloadJob.submit();
while (!isCompleted(infraAppState) && !isCompleted(workloadAppState)) { while (!Apps.isApplicationFinalState(infraAppState) &&
!isCompleted(workloadAppState)) {
workloadJob.monitorAndPrintJob(); workloadJob.monitorAndPrintJob();
Thread.sleep(5000); Thread.sleep(5000);
workloadAppState = workloadJob.getJobState(); workloadAppState = workloadJob.getJobState();
@ -1093,7 +1096,7 @@ public void attemptCleanup() {
} }
} }
} }
if (infraAppId != null && !isCompleted(infraAppState)) { if (infraAppId != null && !Apps.isApplicationFinalState(infraAppState)) {
try { try {
LOG.info("Attempting to kill infrastructure app: " + infraAppId); LOG.info("Attempting to kill infrastructure app: " + infraAppId);
forceKillApplication(infraAppId); forceKillApplication(infraAppId);
@ -1112,15 +1115,6 @@ private static boolean isCompleted(JobStatus.State state) {
|| state == JobStatus.State.KILLED; || state == JobStatus.State.KILLED;
} }
/**
* Check if the input state represents completion.
*/
private static boolean isCompleted(YarnApplicationState state) {
return state == YarnApplicationState.FINISHED
|| state == YarnApplicationState.FAILED
|| state == YarnApplicationState.KILLED;
}
/** /**
* Kill a submitted application by sending a call to the AM. * Kill a submitted application by sending a call to the AM.
* *

View File

@ -57,6 +57,7 @@
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException; import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.util.Times;
import com.google.common.annotations.VisibleForTesting; import com.google.common.annotations.VisibleForTesting;
@ -545,9 +546,7 @@ private void killApplication(String applicationId) throws YarnException,
throw e; throw e;
} }
if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED if (Apps.isApplicationFinalState(appReport.getYarnApplicationState())) {
|| appReport.getYarnApplicationState() == YarnApplicationState.KILLED
|| appReport.getYarnApplicationState() == YarnApplicationState.FAILED) {
sysout.println("Application " + applicationId + " has already finished "); sysout.println("Application " + applicationId + " has already finished ");
} else { } else {
sysout.println("Killing application " + applicationId); sysout.println("Killing application " + applicationId);
@ -562,9 +561,7 @@ private void moveApplicationAcrossQueues(String applicationId, String queue)
throws YarnException, IOException { throws YarnException, IOException {
ApplicationId appId = ApplicationId.fromString(applicationId); ApplicationId appId = ApplicationId.fromString(applicationId);
ApplicationReport appReport = client.getApplicationReport(appId); ApplicationReport appReport = client.getApplicationReport(appId);
if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED if (Apps.isApplicationFinalState(appReport.getYarnApplicationState())) {
|| appReport.getYarnApplicationState() == YarnApplicationState.KILLED
|| appReport.getYarnApplicationState() == YarnApplicationState.FAILED) {
sysout.println("Application " + applicationId + " has already finished "); sysout.println("Application " + applicationId + " has already finished ");
} else { } else {
sysout.println("Moving application " + applicationId + " to queue " + queue); sysout.println("Moving application " + applicationId + " to queue " + queue);

View File

@ -87,6 +87,7 @@
import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers; import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers;
import org.apache.hadoop.yarn.logaggregation.LogToolUtils; import org.apache.hadoop.yarn.logaggregation.LogToolUtils;
import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants; import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils; import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONArray;
@ -356,8 +357,8 @@ private int runCommand(String[] args) throws Exception {
ContainerLogsRequest request = new ContainerLogsRequest(appId, ContainerLogsRequest request = new ContainerLogsRequest(appId,
isApplicationFinished(appState), appOwner, nodeAddress, null, Apps.isApplicationFinalState(appState), appOwner, nodeAddress,
containerIdStr, localDir, logs, bytes, null); null, containerIdStr, localDir, logs, bytes, null);
if (showContainerLogInfo) { if (showContainerLogInfo) {
return showContainerLogInfo(request, logCliHelper); return showContainerLogInfo(request, logCliHelper);
@ -597,7 +598,7 @@ public int printContainerLogsFromRunningApplication(Configuration conf,
for (String logFile : request.getLogTypes()) { for (String logFile : request.getLogTypes()) {
InputStream is = null; InputStream is = null;
try { try {
ClientResponse response = getResponeFromNMWebService(conf, ClientResponse response = getResponseFromNMWebService(conf,
webServiceClient, request, logFile); webServiceClient, request, logFile);
if (response != null && response.getStatusInfo().getStatusCode() == if (response != null && response.getStatusInfo().getStatusCode() ==
ClientResponse.Status.OK.getStatusCode()) { ClientResponse.Status.OK.getStatusCode()) {
@ -645,12 +646,6 @@ public ContainerReport getContainerReport(String containerIdStr)
ContainerId.fromString(containerIdStr)); ContainerId.fromString(containerIdStr));
} }
private boolean isApplicationFinished(YarnApplicationState appState) {
return appState == YarnApplicationState.FINISHED
|| appState == YarnApplicationState.FAILED
|| appState == YarnApplicationState.KILLED;
}
private int printAMContainerLogs(Configuration conf, private int printAMContainerLogs(Configuration conf,
ContainerLogsRequest request, List<String> amContainers, ContainerLogsRequest request, List<String> amContainers,
LogCLIHelpers logCliHelper, boolean useRegex, boolean ignoreSizeLimit) LogCLIHelpers logCliHelper, boolean useRegex, boolean ignoreSizeLimit)
@ -1462,16 +1457,10 @@ public Set<String> getMatchedContainerLogFiles(ContainerLogsRequest request,
} }
@VisibleForTesting @VisibleForTesting
public ClientResponse getResponeFromNMWebService(Configuration conf, public ClientResponse getResponseFromNMWebService(Configuration conf,
Client webServiceClient, ContainerLogsRequest request, String logFile) { Client webServiceClient, ContainerLogsRequest request, String logFile) {
WebResource webResource = return LogToolUtils.getResponseFromNMWebService(
webServiceClient.resource(WebAppUtils.getHttpSchemePrefix(conf) conf, webServiceClient, request, logFile);
+ request.getNodeHttpAddress());
return webResource.path("ws").path("v1").path("node")
.path("containers").path(request.getContainerId()).path("logs")
.path(logFile)
.queryParam("size", Long.toString(request.getBytes()))
.accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
} }
@VisibleForTesting @VisibleForTesting

View File

@ -840,7 +840,7 @@ public void testGetRunningContainerLogs() throws Exception {
ClientResponse mockReponse = mock(ClientResponse.class); ClientResponse mockReponse = mock(ClientResponse.class);
doReturn(Status.OK).when(mockReponse).getStatusInfo(); doReturn(Status.OK).when(mockReponse).getStatusInfo();
doReturn(fis).when(mockReponse).getEntityInputStream(); doReturn(fis).when(mockReponse).getEntityInputStream();
doReturn(mockReponse).when(cli).getResponeFromNMWebService( doReturn(mockReponse).when(cli).getResponseFromNMWebService(
any(Configuration.class), any(Configuration.class),
any(Client.class), any(Client.class),
any(ContainerLogsRequest.class), anyString()); any(ContainerLogsRequest.class), anyString());

View File

@ -24,6 +24,7 @@
import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory; import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory;
import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController; import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController;
import org.apache.hadoop.yarn.util.Apps;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
@ -43,7 +44,6 @@
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ConverterUtils;
import com.google.common.annotations.VisibleForTesting; import com.google.common.annotations.VisibleForTesting;
@ -195,9 +195,7 @@ private static boolean isApplicationTerminated(ApplicationId appId,
throw new IOException(e); throw new IOException(e);
} }
YarnApplicationState currentState = appReport.getYarnApplicationState(); YarnApplicationState currentState = appReport.getYarnApplicationState();
return currentState == YarnApplicationState.FAILED return Apps.isApplicationFinalState(currentState);
|| currentState == YarnApplicationState.KILLED
|| currentState == YarnApplicationState.FINISHED;
} }
public ApplicationClientProtocol getRMClient() { public ApplicationClientProtocol getRMClient() {

View File

@ -28,8 +28,16 @@
import java.nio.charset.Charset; import java.nio.charset.Charset;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Paths; import java.nio.file.Paths;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import javax.ws.rs.core.MediaType;
/** /**
* This class contains several utility function which could be used in different * This class contains several utility function which could be used in different
@ -182,4 +190,26 @@ public static PrintStream createPrintStream(String localDir, String nodeId,
} }
return out; return out;
} }
/**
* Redirect the {@link ContainerLogsRequest} to the NodeManager's
* NMWebServices.
*
* @param conf Configuration object
* @param webServiceClient client
* @param request the request for container logs
* @param logFile name of the log file
* @return response from NMWebServices
*/
public static ClientResponse getResponseFromNMWebService(Configuration conf,
Client webServiceClient, ContainerLogsRequest request, String logFile) {
WebResource webResource =
webServiceClient.resource(WebAppUtils.getHttpSchemePrefix(conf)
+ request.getNodeHttpAddress());
return webResource.path("ws").path("v1").path("node")
.path("containers").path(request.getContainerId()).path("logs")
.path(logFile)
.queryParam("size", Long.toString(request.getBytes()))
.accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
}
} }

View File

@ -40,6 +40,7 @@
import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus; import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
/** /**
@ -301,4 +302,17 @@ public static boolean shouldCountTowardsNodeBlacklisting(int exitStatus) {
return true; return true;
} }
} }
/**
* Returns whether a given application state is final: FINISHED,
* FAILED or KILLED.
*
* @param appState application state
* @return whether the appState is final.
*/
public static boolean isApplicationFinalState(YarnApplicationState appState) {
return appState == YarnApplicationState.FINISHED
|| appState == YarnApplicationState.FAILED
|| appState == YarnApplicationState.KILLED;
}
} }

View File

@ -64,7 +64,7 @@
import org.apache.hadoop.yarn.server.timeline.NameValuePair; import org.apache.hadoop.yarn.server.timeline.NameValuePair;
import org.apache.hadoop.yarn.server.timeline.TimelineDataManager; import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field; import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field;
import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import com.google.common.annotations.VisibleForTesting; import com.google.common.annotations.VisibleForTesting;
@ -412,7 +412,7 @@ private static ApplicationReportExt convertToApplicationReport(
} }
if (eventInfo.containsKey( if (eventInfo.containsKey(
ApplicationMetricsConstants.STATE_EVENT_INFO)) { ApplicationMetricsConstants.STATE_EVENT_INFO)) {
if (!isFinalState(state)) { if (!Apps.isApplicationFinalState(state)) {
state = YarnApplicationState.valueOf(eventInfo.get( state = YarnApplicationState.valueOf(eventInfo.get(
ApplicationMetricsConstants.STATE_EVENT_INFO).toString()); ApplicationMetricsConstants.STATE_EVENT_INFO).toString());
} }
@ -475,12 +475,6 @@ private static long parseLong(Map<String, Object> entityInfo,
return result; return result;
} }
private static boolean isFinalState(YarnApplicationState state) {
return state == YarnApplicationState.FINISHED
|| state == YarnApplicationState.FAILED
|| state == YarnApplicationState.KILLED;
}
private static ApplicationAttemptReport convertToApplicationAttemptReport( private static ApplicationAttemptReport convertToApplicationAttemptReport(
TimelineEntity entity) { TimelineEntity entity) {
String host = null; String host = null;

View File

@ -176,7 +176,7 @@ public Collection<ApplicationAttemptReport> run()
&& conf.getBoolean(YarnConfiguration.RM_WEBAPP_UI_ACTIONS_ENABLED, && conf.getBoolean(YarnConfiguration.RM_WEBAPP_UI_ACTIONS_ENABLED,
YarnConfiguration.DEFAULT_RM_WEBAPP_UI_ACTIONS_ENABLED) YarnConfiguration.DEFAULT_RM_WEBAPP_UI_ACTIONS_ENABLED)
&& !unsecuredUIForSecuredCluster && !unsecuredUIForSecuredCluster
&& !isAppInFinalState(app)) { && !Apps.isApplicationFinalState(app.getAppState())) {
// Application Kill // Application Kill
html.div() html.div()
.button() .button()
@ -253,11 +253,9 @@ private void generateOverviewTable(AppInfo app, String schedulerPath,
|| app.getTrackingUrl().equals(UNAVAILABLE) ? null : root_url(app || app.getTrackingUrl().equals(UNAVAILABLE) ? null : root_url(app
.getTrackingUrl()), .getTrackingUrl()),
app.getTrackingUrl() == null app.getTrackingUrl() == null
|| app.getTrackingUrl().equals(UNAVAILABLE) ? "Unassigned" : app || app.getTrackingUrl().equals(UNAVAILABLE) ? "Unassigned" :
.getAppState() == YarnApplicationState.FINISHED Apps.isApplicationFinalState(app.getAppState()) ?
|| app.getAppState() == YarnApplicationState.FAILED "History" : "ApplicationMaster");
|| app.getAppState() == YarnApplicationState.KILLED ? "History"
: "ApplicationMaster");
if (webUiType != null if (webUiType != null
&& webUiType.equals(YarnWebParams.RM_WEB_UI)) { && webUiType.equals(YarnWebParams.RM_WEB_UI)) {
LogAggregationStatus status = getLogAggregationStatus(); LogAggregationStatus status = getLogAggregationStatus();
@ -448,10 +446,4 @@ public static String getCSRFHeaderString(Configuration conf) {
} }
return ret; return ret;
} }
private boolean isAppInFinalState(AppInfo app) {
return app.getAppState() == YarnApplicationState.FINISHED
|| app.getAppState() == YarnApplicationState.FAILED
|| app.getAppState() == YarnApplicationState.KILLED;
}
} }

View File

@ -42,6 +42,7 @@
import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.webapp.dao.AppInfo; import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.BadRequestException;
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE;
@ -211,10 +212,8 @@ protected void renderData(Block html) {
String trackingUI = String trackingUI =
app.getTrackingUrl() == null || app.getTrackingUrl().equals(UNAVAILABLE) app.getTrackingUrl() == null || app.getTrackingUrl().equals(UNAVAILABLE)
? "Unassigned" ? "Unassigned" :
: app.getAppState() == YarnApplicationState.FINISHED Apps.isApplicationFinalState(app.getAppState())
|| app.getAppState() == YarnApplicationState.FAILED
|| app.getAppState() == YarnApplicationState.KILLED
? "History" : "ApplicationMaster"; ? "History" : "ApplicationMaster";
appsTableData.append(trackingURL == null ? "#" : "href='" + trackingURL) appsTableData.append(trackingURL == null ? "#" : "href='" + trackingURL)
.append("'>").append(trackingUI).append("</a>\"],\n"); .append("'>").append(trackingUI).append("</a>\"],\n");

View File

@ -27,6 +27,7 @@
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory; import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory;
import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.BadRequestException;
import org.apache.hadoop.yarn.webapp.NotFoundException; import org.apache.hadoop.yarn.webapp.NotFoundException;
import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONException;
@ -97,7 +98,7 @@ public Response getContainerLogsInfo(HttpServletRequest req,
} }
// if the application finishes, directly find logs // if the application finishes, directly find logs
// from HDFS. // from HDFS.
if (LogWebServiceUtils.isFinishedState(appInfo.getAppState())) { if (Apps.isApplicationFinalState(appInfo.getAppState())) {
return LogWebServiceUtils return LogWebServiceUtils
.getContainerLogMeta(factory, appId, null, null, containerIdStr, .getContainerLogMeta(factory, appId, null, null, containerIdStr,
false); false);
@ -197,7 +198,7 @@ public Response getLogFile(HttpServletRequest req, String containerIdStr,
filename, format, length, false); filename, format, length, false);
} }
String appOwner = appInfo.getUser(); String appOwner = appInfo.getUser();
if (LogWebServiceUtils.isFinishedState(appInfo.getAppState())) { if (Apps.isApplicationFinalState(appInfo.getAppState())) {
// directly find logs from HDFS. // directly find logs from HDFS.
return LogWebServiceUtils return LogWebServiceUtils
.sendStreamOutputResponse(factory, appId, appOwner, null, .sendStreamOutputResponse(factory, appId, appOwner, null,

View File

@ -222,12 +222,6 @@ public static boolean isRunningState(YarnApplicationState appState) {
return appState == YarnApplicationState.RUNNING; return appState == YarnApplicationState.RUNNING;
} }
public static boolean isFinishedState(YarnApplicationState appState) {
return appState == YarnApplicationState.FINISHED
|| appState == YarnApplicationState.FAILED
|| appState == YarnApplicationState.KILLED;
}
protected static UserGroupInformation getUser(HttpServletRequest req) { protected static UserGroupInformation getUser(HttpServletRequest req) {
String remoteUser = req.getRemoteUser(); String remoteUser = req.getRemoteUser();
UserGroupInformation callerUGI = null; UserGroupInformation callerUGI = null;

View File

@ -38,6 +38,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.webapp.AppsBlock; import org.apache.hadoop.yarn.server.webapp.AppsBlock;
import org.apache.hadoop.yarn.server.webapp.dao.AppInfo; import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.webapp.View; import org.apache.hadoop.yarn.webapp.View;
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE;
@ -197,10 +198,8 @@ protected void renderData(Block html) {
app.getTrackingUrl() == null app.getTrackingUrl() == null
|| app.getTrackingUrl().equals(UNAVAILABLE) || app.getTrackingUrl().equals(UNAVAILABLE)
|| app.getAppState() == YarnApplicationState.NEW ? "Unassigned" || app.getAppState() == YarnApplicationState.NEW ? "Unassigned"
: app.getAppState() == YarnApplicationState.FINISHED : Apps.isApplicationFinalState(app.getAppState()) ?
|| app.getAppState() == YarnApplicationState.FAILED "History" : "ApplicationMaster";
|| app.getAppState() == YarnApplicationState.KILLED ? "History"
: "ApplicationMaster";
appsTableData.append(trackingURL == null ? "#" : "href='" + trackingURL) appsTableData.append(trackingURL == null ? "#" : "href='" + trackingURL)
.append("'>").append(trackingUI).append("</a>\",").append("\"") .append("'>").append(trackingUI).append("</a>\",").append("\"")
.append(blacklistedNodesCount).append("\"],\n"); .append(blacklistedNodesCount).append("\"],\n");

View File

@ -42,7 +42,6 @@
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain; import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains; import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
@ -56,6 +55,7 @@
import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.timeline.TimelineDataManager.CheckAcl; import org.apache.hadoop.yarn.server.timeline.TimelineDataManager.CheckAcl;
import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager; import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager;
import org.apache.hadoop.yarn.util.Apps;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -104,11 +104,6 @@ public class EntityGroupFSTimelineStore extends CompositeService
private static final FsPermission DONE_DIR_PERMISSION = private static final FsPermission DONE_DIR_PERMISSION =
new FsPermission((short) 0700); new FsPermission((short) 0700);
private static final EnumSet<YarnApplicationState>
APP_FINAL_STATES = EnumSet.of(
YarnApplicationState.FAILED,
YarnApplicationState.KILLED,
YarnApplicationState.FINISHED);
// Active dir: <activeRoot>/appId/attemptId/cacheId.log // Active dir: <activeRoot>/appId/attemptId/cacheId.log
// Done dir: <doneRoot>/cluster_ts/hash1/hash2/appId/attemptId/cacheId.log // Done dir: <doneRoot>/cluster_ts/hash1/hash2/appId/attemptId/cacheId.log
private static final String APP_DONE_DIR_PREFIX_FORMAT = private static final String APP_DONE_DIR_PREFIX_FORMAT =
@ -649,8 +644,7 @@ private static synchronized AppState getAppState(ApplicationId appId,
AppState appState = AppState.ACTIVE; AppState appState = AppState.ACTIVE;
try { try {
ApplicationReport report = yarnClient.getApplicationReport(appId); ApplicationReport report = yarnClient.getApplicationReport(appId);
YarnApplicationState yarnState = report.getYarnApplicationState(); if (Apps.isApplicationFinalState(report.getYarnApplicationState())) {
if (APP_FINAL_STATES.contains(yarnState)) {
appState = AppState.COMPLETED; appState = AppState.COMPLETED;
} }
} catch (ApplicationNotFoundException e) { } catch (ApplicationNotFoundException e) {