diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java index 152c8af194..89de9bd494 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java @@ -69,6 +69,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.util.Apps; public class NotRunningJob implements MRClientProtocol { @@ -138,9 +139,7 @@ public GetJobReportResponse getJobReport(GetJobReportRequest request) jobReport.setUser(applicationReport.getUser()); jobReport.setStartTime(applicationReport.getStartTime()); YarnApplicationState state = applicationReport.getYarnApplicationState(); - if (state == YarnApplicationState.KILLED - || state == YarnApplicationState.FAILED - || state == YarnApplicationState.FINISHED) { + if (Apps.isApplicationFinalState(state)) { jobReport.setDiagnostics(applicationReport.getDiagnostics()); } jobReport.setJobName(applicationReport.getName()); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java index 7d33ed23d9..a9356e28a7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java @@ -94,6 +94,7 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.security.client.RMDelegationTokenSelector; +import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.UnitsConversionUtil; import org.apache.hadoop.yarn.util.resource.ResourceUtils; @@ -899,9 +900,7 @@ private void killUnFinishedApplication(ApplicationId appId) } catch (YarnException e) { throw new IOException(e); } - if (application.getYarnApplicationState() == YarnApplicationState.FINISHED - || application.getYarnApplicationState() == YarnApplicationState.FAILED - || application.getYarnApplicationState() == YarnApplicationState.KILLED) { + if (Apps.isApplicationFinalState(application.getYarnApplicationState())) { return; } killApplication(appId); diff --git a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/Client.java b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/Client.java index 36f90b54d3..3c735dd1cf 100644 --- a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/Client.java +++ b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/java/org/apache/hadoop/tools/dynamometer/Client.java @@ -96,6 +96,7 @@ import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.util.Records; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -891,7 +892,8 @@ private boolean monitorInfraApplication() throws YarnException, IOException { boolean success = false; Thread namenodeMonitoringThread = new Thread(() -> { - Supplier exitCritera = () -> isCompleted(infraAppState); + Supplier exitCritera = () -> + Apps.isApplicationFinalState(infraAppState); Optional namenodeProperties = Optional.empty(); while (!exitCritera.get()) { try { @@ -925,7 +927,7 @@ private boolean monitorInfraApplication() throws YarnException, IOException { return; } } - if (!isCompleted(infraAppState) && launchWorkloadJob) { + if (!Apps.isApplicationFinalState(infraAppState) && launchWorkloadJob) { launchAndMonitorWorkloadDriver(namenodeProperties.get()); } }); @@ -1050,7 +1052,8 @@ private void launchAndMonitorWorkloadDriver(Properties nameNodeProperties) { workloadJob = WorkloadDriver.getJobForSubmission(workloadConf, nameNodeURI.toString(), workloadStartTime, AuditReplayMapper.class); workloadJob.submit(); - while (!isCompleted(infraAppState) && !isCompleted(workloadAppState)) { + while (!Apps.isApplicationFinalState(infraAppState) && + !isCompleted(workloadAppState)) { workloadJob.monitorAndPrintJob(); Thread.sleep(5000); workloadAppState = workloadJob.getJobState(); @@ -1093,7 +1096,7 @@ public void attemptCleanup() { } } } - if (infraAppId != null && !isCompleted(infraAppState)) { + if (infraAppId != null && !Apps.isApplicationFinalState(infraAppState)) { try { LOG.info("Attempting to kill infrastructure app: " + infraAppId); forceKillApplication(infraAppId); @@ -1112,15 +1115,6 @@ private static boolean isCompleted(JobStatus.State state) { || state == JobStatus.State.KILLED; } - /** - * Check if the input state represents completion. - */ - private static boolean isCompleted(YarnApplicationState state) { - return state == YarnApplicationState.FINISHED - || state == YarnApplicationState.FAILED - || state == YarnApplicationState.KILLED; - } - /** * Kill a submitted application by sending a call to the AM. * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index 6ec484876b..cfb3b2923d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.util.Times; import com.google.common.annotations.VisibleForTesting; @@ -545,9 +546,7 @@ private void killApplication(String applicationId) throws YarnException, throw e; } - if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED - || appReport.getYarnApplicationState() == YarnApplicationState.KILLED - || appReport.getYarnApplicationState() == YarnApplicationState.FAILED) { + if (Apps.isApplicationFinalState(appReport.getYarnApplicationState())) { sysout.println("Application " + applicationId + " has already finished "); } else { sysout.println("Killing application " + applicationId); @@ -562,9 +561,7 @@ private void moveApplicationAcrossQueues(String applicationId, String queue) throws YarnException, IOException { ApplicationId appId = ApplicationId.fromString(applicationId); ApplicationReport appReport = client.getApplicationReport(appId); - if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED - || appReport.getYarnApplicationState() == YarnApplicationState.KILLED - || appReport.getYarnApplicationState() == YarnApplicationState.FAILED) { + if (Apps.isApplicationFinalState(appReport.getYarnApplicationState())) { sysout.println("Application " + applicationId + " has already finished "); } else { sysout.println("Moving application " + applicationId + " to queue " + queue); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index 2b5439bf53..d52f6ad23a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -87,6 +87,7 @@ import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers; import org.apache.hadoop.yarn.logaggregation.LogToolUtils; import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants; +import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils; import org.codehaus.jettison.json.JSONArray; @@ -356,8 +357,8 @@ private int runCommand(String[] args) throws Exception { ContainerLogsRequest request = new ContainerLogsRequest(appId, - isApplicationFinished(appState), appOwner, nodeAddress, null, - containerIdStr, localDir, logs, bytes, null); + Apps.isApplicationFinalState(appState), appOwner, nodeAddress, + null, containerIdStr, localDir, logs, bytes, null); if (showContainerLogInfo) { return showContainerLogInfo(request, logCliHelper); @@ -597,7 +598,7 @@ public int printContainerLogsFromRunningApplication(Configuration conf, for (String logFile : request.getLogTypes()) { InputStream is = null; try { - ClientResponse response = getResponeFromNMWebService(conf, + ClientResponse response = getResponseFromNMWebService(conf, webServiceClient, request, logFile); if (response != null && response.getStatusInfo().getStatusCode() == ClientResponse.Status.OK.getStatusCode()) { @@ -645,12 +646,6 @@ public ContainerReport getContainerReport(String containerIdStr) ContainerId.fromString(containerIdStr)); } - private boolean isApplicationFinished(YarnApplicationState appState) { - return appState == YarnApplicationState.FINISHED - || appState == YarnApplicationState.FAILED - || appState == YarnApplicationState.KILLED; - } - private int printAMContainerLogs(Configuration conf, ContainerLogsRequest request, List amContainers, LogCLIHelpers logCliHelper, boolean useRegex, boolean ignoreSizeLimit) @@ -1462,16 +1457,10 @@ public Set getMatchedContainerLogFiles(ContainerLogsRequest request, } @VisibleForTesting - public ClientResponse getResponeFromNMWebService(Configuration conf, + public ClientResponse getResponseFromNMWebService(Configuration conf, Client webServiceClient, ContainerLogsRequest request, String logFile) { - WebResource webResource = - webServiceClient.resource(WebAppUtils.getHttpSchemePrefix(conf) - + request.getNodeHttpAddress()); - return webResource.path("ws").path("v1").path("node") - .path("containers").path(request.getContainerId()).path("logs") - .path(logFile) - .queryParam("size", Long.toString(request.getBytes())) - .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); + return LogToolUtils.getResponseFromNMWebService( + conf, webServiceClient, request, logFile); } @VisibleForTesting diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index 03bad0d642..7966313b68 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -840,7 +840,7 @@ public void testGetRunningContainerLogs() throws Exception { ClientResponse mockReponse = mock(ClientResponse.class); doReturn(Status.OK).when(mockReponse).getStatusInfo(); doReturn(fis).when(mockReponse).getEntityInputStream(); - doReturn(mockReponse).when(cli).getResponeFromNMWebService( + doReturn(mockReponse).when(cli).getResponseFromNMWebService( any(Configuration.class), any(Client.class), any(ContainerLogsRequest.class), anyString()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java index e0233b3164..a7774f6bd8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java @@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory; import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController; +import org.apache.hadoop.yarn.util.Apps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -43,7 +44,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.util.ConverterUtils; import com.google.common.annotations.VisibleForTesting; @@ -195,9 +195,7 @@ private static boolean isApplicationTerminated(ApplicationId appId, throw new IOException(e); } YarnApplicationState currentState = appReport.getYarnApplicationState(); - return currentState == YarnApplicationState.FAILED - || currentState == YarnApplicationState.KILLED - || currentState == YarnApplicationState.FINISHED; + return Apps.isApplicationFinalState(currentState); } public ApplicationClientProtocol getRMClient() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogToolUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogToolUtils.java index 5bc0b14e69..3c56b0290d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogToolUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogToolUtils.java @@ -28,8 +28,16 @@ import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Paths; + +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; + +import javax.ws.rs.core.MediaType; /** * This class contains several utility function which could be used in different @@ -182,4 +190,26 @@ public static PrintStream createPrintStream(String localDir, String nodeId, } return out; } + + /** + * Redirect the {@link ContainerLogsRequest} to the NodeManager's + * NMWebServices. + * + * @param conf Configuration object + * @param webServiceClient client + * @param request the request for container logs + * @param logFile name of the log file + * @return response from NMWebServices + */ + public static ClientResponse getResponseFromNMWebService(Configuration conf, + Client webServiceClient, ContainerLogsRequest request, String logFile) { + WebResource webResource = + webServiceClient.resource(WebAppUtils.getHttpSchemePrefix(conf) + + request.getNodeHttpAddress()); + return webResource.path("ws").path("v1").path("node") + .path("containers").path(request.getContainerId()).path("logs") + .path(logFile) + .queryParam("size", Long.toString(request.getBytes())) + .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java index 1ea7646e32..a53ae03978 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java @@ -40,6 +40,7 @@ import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerExitStatus; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; /** @@ -301,4 +302,17 @@ public static boolean shouldCountTowardsNodeBlacklisting(int exitStatus) { return true; } } + + /** + * Returns whether a given application state is final: FINISHED, + * FAILED or KILLED. + * + * @param appState application state + * @return whether the appState is final. + */ + public static boolean isApplicationFinalState(YarnApplicationState appState) { + return appState == YarnApplicationState.FINISHED + || appState == YarnApplicationState.FAILED + || appState == YarnApplicationState.KILLED; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index cc277ed24a..8afcd6aa45 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -64,7 +64,7 @@ import org.apache.hadoop.yarn.server.timeline.NameValuePair; import org.apache.hadoop.yarn.server.timeline.TimelineDataManager; import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field; -import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import com.google.common.annotations.VisibleForTesting; @@ -412,7 +412,7 @@ private static ApplicationReportExt convertToApplicationReport( } if (eventInfo.containsKey( ApplicationMetricsConstants.STATE_EVENT_INFO)) { - if (!isFinalState(state)) { + if (!Apps.isApplicationFinalState(state)) { state = YarnApplicationState.valueOf(eventInfo.get( ApplicationMetricsConstants.STATE_EVENT_INFO).toString()); } @@ -475,12 +475,6 @@ private static long parseLong(Map entityInfo, return result; } - private static boolean isFinalState(YarnApplicationState state) { - return state == YarnApplicationState.FINISHED - || state == YarnApplicationState.FAILED - || state == YarnApplicationState.KILLED; - } - private static ApplicationAttemptReport convertToApplicationAttemptReport( TimelineEntity entity) { String host = null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java index e09977dc79..e734c2443a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java @@ -176,7 +176,7 @@ public Collection run() && conf.getBoolean(YarnConfiguration.RM_WEBAPP_UI_ACTIONS_ENABLED, YarnConfiguration.DEFAULT_RM_WEBAPP_UI_ACTIONS_ENABLED) && !unsecuredUIForSecuredCluster - && !isAppInFinalState(app)) { + && !Apps.isApplicationFinalState(app.getAppState())) { // Application Kill html.div() .button() @@ -253,11 +253,9 @@ private void generateOverviewTable(AppInfo app, String schedulerPath, || app.getTrackingUrl().equals(UNAVAILABLE) ? null : root_url(app .getTrackingUrl()), app.getTrackingUrl() == null - || app.getTrackingUrl().equals(UNAVAILABLE) ? "Unassigned" : app - .getAppState() == YarnApplicationState.FINISHED - || app.getAppState() == YarnApplicationState.FAILED - || app.getAppState() == YarnApplicationState.KILLED ? "History" - : "ApplicationMaster"); + || app.getTrackingUrl().equals(UNAVAILABLE) ? "Unassigned" : + Apps.isApplicationFinalState(app.getAppState()) ? + "History" : "ApplicationMaster"); if (webUiType != null && webUiType.equals(YarnWebParams.RM_WEB_UI)) { LogAggregationStatus status = getLogAggregationStatus(); @@ -448,10 +446,4 @@ public static String getCSRFHeaderString(Configuration conf) { } return ret; } - - private boolean isAppInFinalState(AppInfo app) { - return app.getAppState() == YarnApplicationState.FINISHED - || app.getAppState() == YarnApplicationState.FAILED - || app.getAppState() == YarnApplicationState.KILLED; - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java index 29843b53ab..b89ca02e09 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java @@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.webapp.dao.AppInfo; +import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE; @@ -211,10 +212,8 @@ protected void renderData(Block html) { String trackingUI = app.getTrackingUrl() == null || app.getTrackingUrl().equals(UNAVAILABLE) - ? "Unassigned" - : app.getAppState() == YarnApplicationState.FINISHED - || app.getAppState() == YarnApplicationState.FAILED - || app.getAppState() == YarnApplicationState.KILLED + ? "Unassigned" : + Apps.isApplicationFinalState(app.getAppState()) ? "History" : "ApplicationMaster"; appsTableData.append(trackingURL == null ? "#" : "href='" + trackingURL) .append("'>").append(trackingUI).append("\"],\n"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java index 1bddf6dfda..39e0ae308b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogServlet.java @@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory; +import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.NotFoundException; import org.codehaus.jettison.json.JSONException; @@ -97,7 +98,7 @@ public Response getContainerLogsInfo(HttpServletRequest req, } // if the application finishes, directly find logs // from HDFS. - if (LogWebServiceUtils.isFinishedState(appInfo.getAppState())) { + if (Apps.isApplicationFinalState(appInfo.getAppState())) { return LogWebServiceUtils .getContainerLogMeta(factory, appId, null, null, containerIdStr, false); @@ -197,7 +198,7 @@ public Response getLogFile(HttpServletRequest req, String containerIdStr, filename, format, length, false); } String appOwner = appInfo.getUser(); - if (LogWebServiceUtils.isFinishedState(appInfo.getAppState())) { + if (Apps.isApplicationFinalState(appInfo.getAppState())) { // directly find logs from HDFS. return LogWebServiceUtils .sendStreamOutputResponse(factory, appId, appOwner, null, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebServiceUtils.java index bc301bbb2d..defde4d39a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebServiceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/LogWebServiceUtils.java @@ -222,12 +222,6 @@ public static boolean isRunningState(YarnApplicationState appState) { return appState == YarnApplicationState.RUNNING; } - public static boolean isFinishedState(YarnApplicationState appState) { - return appState == YarnApplicationState.FINISHED - || appState == YarnApplicationState.FAILED - || appState == YarnApplicationState.KILLED; - } - protected static UserGroupInformation getUser(HttpServletRequest req) { String remoteUser = req.getRemoteUser(); UserGroupInformation callerUGI = null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java index 8868fa1093..e8da0cf9ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java @@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.webapp.AppsBlock; import org.apache.hadoop.yarn.server.webapp.dao.AppInfo; +import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.webapp.View; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE; @@ -197,10 +198,8 @@ protected void renderData(Block html) { app.getTrackingUrl() == null || app.getTrackingUrl().equals(UNAVAILABLE) || app.getAppState() == YarnApplicationState.NEW ? "Unassigned" - : app.getAppState() == YarnApplicationState.FINISHED - || app.getAppState() == YarnApplicationState.FAILED - || app.getAppState() == YarnApplicationState.KILLED ? "History" - : "ApplicationMaster"; + : Apps.isApplicationFinalState(app.getAppState()) ? + "History" : "ApplicationMaster"; appsTableData.append(trackingURL == null ? "#" : "href='" + trackingURL) .append("'>").append(trackingUI).append("\",").append("\"") .append(blacklistedNodesCount).append("\"],\n"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java index 8f00b852c0..beff2751d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/EntityGroupFSTimelineStore.java @@ -42,7 +42,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain; import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; @@ -56,6 +55,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.timeline.TimelineDataManager.CheckAcl; import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager; +import org.apache.hadoop.yarn.util.Apps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -104,11 +104,6 @@ public class EntityGroupFSTimelineStore extends CompositeService private static final FsPermission DONE_DIR_PERMISSION = new FsPermission((short) 0700); - private static final EnumSet - APP_FINAL_STATES = EnumSet.of( - YarnApplicationState.FAILED, - YarnApplicationState.KILLED, - YarnApplicationState.FINISHED); // Active dir: /appId/attemptId/cacheId.log // Done dir: /cluster_ts/hash1/hash2/appId/attemptId/cacheId.log private static final String APP_DONE_DIR_PREFIX_FORMAT = @@ -649,8 +644,7 @@ private static synchronized AppState getAppState(ApplicationId appId, AppState appState = AppState.ACTIVE; try { ApplicationReport report = yarnClient.getApplicationReport(appId); - YarnApplicationState yarnState = report.getYarnApplicationState(); - if (APP_FINAL_STATES.contains(yarnState)) { + if (Apps.isApplicationFinalState(report.getYarnApplicationState())) { appState = AppState.COMPLETED; } } catch (ApplicationNotFoundException e) {