YARN-10083. Provide utility to ask whether an application is in final status. Contributed by Adam Antal
This commit is contained in:
parent
bd03053ea2
commit
9520b2ad79
@ -69,6 +69,7 @@
|
|||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.factories.RecordFactory;
|
import org.apache.hadoop.yarn.factories.RecordFactory;
|
||||||
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
||||||
|
import org.apache.hadoop.yarn.util.Apps;
|
||||||
|
|
||||||
public class NotRunningJob implements MRClientProtocol {
|
public class NotRunningJob implements MRClientProtocol {
|
||||||
|
|
||||||
@ -138,9 +139,7 @@ public GetJobReportResponse getJobReport(GetJobReportRequest request)
|
|||||||
jobReport.setUser(applicationReport.getUser());
|
jobReport.setUser(applicationReport.getUser());
|
||||||
jobReport.setStartTime(applicationReport.getStartTime());
|
jobReport.setStartTime(applicationReport.getStartTime());
|
||||||
YarnApplicationState state = applicationReport.getYarnApplicationState();
|
YarnApplicationState state = applicationReport.getYarnApplicationState();
|
||||||
if (state == YarnApplicationState.KILLED
|
if (Apps.isApplicationFinalState(state)) {
|
||||||
|| state == YarnApplicationState.FAILED
|
|
||||||
|| state == YarnApplicationState.FINISHED) {
|
|
||||||
jobReport.setDiagnostics(applicationReport.getDiagnostics());
|
jobReport.setDiagnostics(applicationReport.getDiagnostics());
|
||||||
}
|
}
|
||||||
jobReport.setJobName(applicationReport.getName());
|
jobReport.setJobName(applicationReport.getName());
|
||||||
|
@ -94,6 +94,7 @@
|
|||||||
import org.apache.hadoop.yarn.factories.RecordFactory;
|
import org.apache.hadoop.yarn.factories.RecordFactory;
|
||||||
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
||||||
import org.apache.hadoop.yarn.security.client.RMDelegationTokenSelector;
|
import org.apache.hadoop.yarn.security.client.RMDelegationTokenSelector;
|
||||||
|
import org.apache.hadoop.yarn.util.Apps;
|
||||||
import org.apache.hadoop.yarn.util.ConverterUtils;
|
import org.apache.hadoop.yarn.util.ConverterUtils;
|
||||||
import org.apache.hadoop.yarn.util.UnitsConversionUtil;
|
import org.apache.hadoop.yarn.util.UnitsConversionUtil;
|
||||||
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
|
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
|
||||||
@ -899,9 +900,7 @@ private void killUnFinishedApplication(ApplicationId appId)
|
|||||||
} catch (YarnException e) {
|
} catch (YarnException e) {
|
||||||
throw new IOException(e);
|
throw new IOException(e);
|
||||||
}
|
}
|
||||||
if (application.getYarnApplicationState() == YarnApplicationState.FINISHED
|
if (Apps.isApplicationFinalState(application.getYarnApplicationState())) {
|
||||||
|| application.getYarnApplicationState() == YarnApplicationState.FAILED
|
|
||||||
|| application.getYarnApplicationState() == YarnApplicationState.KILLED) {
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
killApplication(appId);
|
killApplication(appId);
|
||||||
|
@ -96,6 +96,7 @@
|
|||||||
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
|
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
|
import org.apache.hadoop.yarn.util.Apps;
|
||||||
import org.apache.hadoop.yarn.util.Records;
|
import org.apache.hadoop.yarn.util.Records;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
@ -891,7 +892,8 @@ private boolean monitorInfraApplication() throws YarnException, IOException {
|
|||||||
boolean success = false;
|
boolean success = false;
|
||||||
|
|
||||||
Thread namenodeMonitoringThread = new Thread(() -> {
|
Thread namenodeMonitoringThread = new Thread(() -> {
|
||||||
Supplier<Boolean> exitCritera = () -> isCompleted(infraAppState);
|
Supplier<Boolean> exitCritera = () ->
|
||||||
|
Apps.isApplicationFinalState(infraAppState);
|
||||||
Optional<Properties> namenodeProperties = Optional.empty();
|
Optional<Properties> namenodeProperties = Optional.empty();
|
||||||
while (!exitCritera.get()) {
|
while (!exitCritera.get()) {
|
||||||
try {
|
try {
|
||||||
@ -925,7 +927,7 @@ private boolean monitorInfraApplication() throws YarnException, IOException {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (!isCompleted(infraAppState) && launchWorkloadJob) {
|
if (!Apps.isApplicationFinalState(infraAppState) && launchWorkloadJob) {
|
||||||
launchAndMonitorWorkloadDriver(namenodeProperties.get());
|
launchAndMonitorWorkloadDriver(namenodeProperties.get());
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
@ -1050,7 +1052,8 @@ private void launchAndMonitorWorkloadDriver(Properties nameNodeProperties) {
|
|||||||
workloadJob = WorkloadDriver.getJobForSubmission(workloadConf,
|
workloadJob = WorkloadDriver.getJobForSubmission(workloadConf,
|
||||||
nameNodeURI.toString(), workloadStartTime, AuditReplayMapper.class);
|
nameNodeURI.toString(), workloadStartTime, AuditReplayMapper.class);
|
||||||
workloadJob.submit();
|
workloadJob.submit();
|
||||||
while (!isCompleted(infraAppState) && !isCompleted(workloadAppState)) {
|
while (!Apps.isApplicationFinalState(infraAppState) &&
|
||||||
|
!isCompleted(workloadAppState)) {
|
||||||
workloadJob.monitorAndPrintJob();
|
workloadJob.monitorAndPrintJob();
|
||||||
Thread.sleep(5000);
|
Thread.sleep(5000);
|
||||||
workloadAppState = workloadJob.getJobState();
|
workloadAppState = workloadJob.getJobState();
|
||||||
@ -1093,7 +1096,7 @@ public void attemptCleanup() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (infraAppId != null && !isCompleted(infraAppState)) {
|
if (infraAppId != null && !Apps.isApplicationFinalState(infraAppState)) {
|
||||||
try {
|
try {
|
||||||
LOG.info("Attempting to kill infrastructure app: " + infraAppId);
|
LOG.info("Attempting to kill infrastructure app: " + infraAppId);
|
||||||
forceKillApplication(infraAppId);
|
forceKillApplication(infraAppId);
|
||||||
@ -1112,15 +1115,6 @@ private static boolean isCompleted(JobStatus.State state) {
|
|||||||
|| state == JobStatus.State.KILLED;
|
|| state == JobStatus.State.KILLED;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if the input state represents completion.
|
|
||||||
*/
|
|
||||||
private static boolean isCompleted(YarnApplicationState state) {
|
|
||||||
return state == YarnApplicationState.FINISHED
|
|
||||||
|| state == YarnApplicationState.FAILED
|
|
||||||
|| state == YarnApplicationState.KILLED;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Kill a submitted application by sending a call to the AM.
|
* Kill a submitted application by sending a call to the AM.
|
||||||
*
|
*
|
||||||
|
@ -57,6 +57,7 @@
|
|||||||
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
|
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
|
||||||
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
|
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
|
import org.apache.hadoop.yarn.util.Apps;
|
||||||
import org.apache.hadoop.yarn.util.Times;
|
import org.apache.hadoop.yarn.util.Times;
|
||||||
|
|
||||||
import com.google.common.annotations.VisibleForTesting;
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
@ -545,9 +546,7 @@ private void killApplication(String applicationId) throws YarnException,
|
|||||||
throw e;
|
throw e;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED
|
if (Apps.isApplicationFinalState(appReport.getYarnApplicationState())) {
|
||||||
|| appReport.getYarnApplicationState() == YarnApplicationState.KILLED
|
|
||||||
|| appReport.getYarnApplicationState() == YarnApplicationState.FAILED) {
|
|
||||||
sysout.println("Application " + applicationId + " has already finished ");
|
sysout.println("Application " + applicationId + " has already finished ");
|
||||||
} else {
|
} else {
|
||||||
sysout.println("Killing application " + applicationId);
|
sysout.println("Killing application " + applicationId);
|
||||||
@ -562,9 +561,7 @@ private void moveApplicationAcrossQueues(String applicationId, String queue)
|
|||||||
throws YarnException, IOException {
|
throws YarnException, IOException {
|
||||||
ApplicationId appId = ApplicationId.fromString(applicationId);
|
ApplicationId appId = ApplicationId.fromString(applicationId);
|
||||||
ApplicationReport appReport = client.getApplicationReport(appId);
|
ApplicationReport appReport = client.getApplicationReport(appId);
|
||||||
if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED
|
if (Apps.isApplicationFinalState(appReport.getYarnApplicationState())) {
|
||||||
|| appReport.getYarnApplicationState() == YarnApplicationState.KILLED
|
|
||||||
|| appReport.getYarnApplicationState() == YarnApplicationState.FAILED) {
|
|
||||||
sysout.println("Application " + applicationId + " has already finished ");
|
sysout.println("Application " + applicationId + " has already finished ");
|
||||||
} else {
|
} else {
|
||||||
sysout.println("Moving application " + applicationId + " to queue " + queue);
|
sysout.println("Moving application " + applicationId + " to queue " + queue);
|
||||||
|
@ -87,6 +87,7 @@
|
|||||||
import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers;
|
import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers;
|
||||||
import org.apache.hadoop.yarn.logaggregation.LogToolUtils;
|
import org.apache.hadoop.yarn.logaggregation.LogToolUtils;
|
||||||
import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
|
import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
|
||||||
|
import org.apache.hadoop.yarn.util.Apps;
|
||||||
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
||||||
import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
|
import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
|
||||||
import org.codehaus.jettison.json.JSONArray;
|
import org.codehaus.jettison.json.JSONArray;
|
||||||
@ -356,8 +357,8 @@ private int runCommand(String[] args) throws Exception {
|
|||||||
|
|
||||||
|
|
||||||
ContainerLogsRequest request = new ContainerLogsRequest(appId,
|
ContainerLogsRequest request = new ContainerLogsRequest(appId,
|
||||||
isApplicationFinished(appState), appOwner, nodeAddress, null,
|
Apps.isApplicationFinalState(appState), appOwner, nodeAddress,
|
||||||
containerIdStr, localDir, logs, bytes, null);
|
null, containerIdStr, localDir, logs, bytes, null);
|
||||||
|
|
||||||
if (showContainerLogInfo) {
|
if (showContainerLogInfo) {
|
||||||
return showContainerLogInfo(request, logCliHelper);
|
return showContainerLogInfo(request, logCliHelper);
|
||||||
@ -597,7 +598,7 @@ public int printContainerLogsFromRunningApplication(Configuration conf,
|
|||||||
for (String logFile : request.getLogTypes()) {
|
for (String logFile : request.getLogTypes()) {
|
||||||
InputStream is = null;
|
InputStream is = null;
|
||||||
try {
|
try {
|
||||||
ClientResponse response = getResponeFromNMWebService(conf,
|
ClientResponse response = getResponseFromNMWebService(conf,
|
||||||
webServiceClient, request, logFile);
|
webServiceClient, request, logFile);
|
||||||
if (response != null && response.getStatusInfo().getStatusCode() ==
|
if (response != null && response.getStatusInfo().getStatusCode() ==
|
||||||
ClientResponse.Status.OK.getStatusCode()) {
|
ClientResponse.Status.OK.getStatusCode()) {
|
||||||
@ -645,12 +646,6 @@ public ContainerReport getContainerReport(String containerIdStr)
|
|||||||
ContainerId.fromString(containerIdStr));
|
ContainerId.fromString(containerIdStr));
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean isApplicationFinished(YarnApplicationState appState) {
|
|
||||||
return appState == YarnApplicationState.FINISHED
|
|
||||||
|| appState == YarnApplicationState.FAILED
|
|
||||||
|| appState == YarnApplicationState.KILLED;
|
|
||||||
}
|
|
||||||
|
|
||||||
private int printAMContainerLogs(Configuration conf,
|
private int printAMContainerLogs(Configuration conf,
|
||||||
ContainerLogsRequest request, List<String> amContainers,
|
ContainerLogsRequest request, List<String> amContainers,
|
||||||
LogCLIHelpers logCliHelper, boolean useRegex, boolean ignoreSizeLimit)
|
LogCLIHelpers logCliHelper, boolean useRegex, boolean ignoreSizeLimit)
|
||||||
@ -1462,16 +1457,10 @@ public Set<String> getMatchedContainerLogFiles(ContainerLogsRequest request,
|
|||||||
}
|
}
|
||||||
|
|
||||||
@VisibleForTesting
|
@VisibleForTesting
|
||||||
public ClientResponse getResponeFromNMWebService(Configuration conf,
|
public ClientResponse getResponseFromNMWebService(Configuration conf,
|
||||||
Client webServiceClient, ContainerLogsRequest request, String logFile) {
|
Client webServiceClient, ContainerLogsRequest request, String logFile) {
|
||||||
WebResource webResource =
|
return LogToolUtils.getResponseFromNMWebService(
|
||||||
webServiceClient.resource(WebAppUtils.getHttpSchemePrefix(conf)
|
conf, webServiceClient, request, logFile);
|
||||||
+ request.getNodeHttpAddress());
|
|
||||||
return webResource.path("ws").path("v1").path("node")
|
|
||||||
.path("containers").path(request.getContainerId()).path("logs")
|
|
||||||
.path(logFile)
|
|
||||||
.queryParam("size", Long.toString(request.getBytes()))
|
|
||||||
.accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@VisibleForTesting
|
@VisibleForTesting
|
||||||
|
@ -840,7 +840,7 @@ public void testGetRunningContainerLogs() throws Exception {
|
|||||||
ClientResponse mockReponse = mock(ClientResponse.class);
|
ClientResponse mockReponse = mock(ClientResponse.class);
|
||||||
doReturn(Status.OK).when(mockReponse).getStatusInfo();
|
doReturn(Status.OK).when(mockReponse).getStatusInfo();
|
||||||
doReturn(fis).when(mockReponse).getEntityInputStream();
|
doReturn(fis).when(mockReponse).getEntityInputStream();
|
||||||
doReturn(mockReponse).when(cli).getResponeFromNMWebService(
|
doReturn(mockReponse).when(cli).getResponseFromNMWebService(
|
||||||
any(Configuration.class),
|
any(Configuration.class),
|
||||||
any(Client.class),
|
any(Client.class),
|
||||||
any(ContainerLogsRequest.class), anyString());
|
any(ContainerLogsRequest.class), anyString());
|
||||||
|
@ -24,6 +24,7 @@
|
|||||||
|
|
||||||
import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory;
|
import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory;
|
||||||
import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController;
|
import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController;
|
||||||
|
import org.apache.hadoop.yarn.util.Apps;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
@ -43,7 +44,6 @@
|
|||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
|
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
import org.apache.hadoop.yarn.util.ConverterUtils;
|
|
||||||
|
|
||||||
import com.google.common.annotations.VisibleForTesting;
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
|
|
||||||
@ -195,9 +195,7 @@ private static boolean isApplicationTerminated(ApplicationId appId,
|
|||||||
throw new IOException(e);
|
throw new IOException(e);
|
||||||
}
|
}
|
||||||
YarnApplicationState currentState = appReport.getYarnApplicationState();
|
YarnApplicationState currentState = appReport.getYarnApplicationState();
|
||||||
return currentState == YarnApplicationState.FAILED
|
return Apps.isApplicationFinalState(currentState);
|
||||||
|| currentState == YarnApplicationState.KILLED
|
|
||||||
|| currentState == YarnApplicationState.FINISHED;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public ApplicationClientProtocol getRMClient() {
|
public ApplicationClientProtocol getRMClient() {
|
||||||
|
@ -28,8 +28,16 @@
|
|||||||
import java.nio.charset.Charset;
|
import java.nio.charset.Charset;
|
||||||
import java.nio.file.Files;
|
import java.nio.file.Files;
|
||||||
import java.nio.file.Paths;
|
import java.nio.file.Paths;
|
||||||
|
|
||||||
|
import com.sun.jersey.api.client.Client;
|
||||||
|
import com.sun.jersey.api.client.ClientResponse;
|
||||||
|
import com.sun.jersey.api.client.WebResource;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
|
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
||||||
|
|
||||||
|
import javax.ws.rs.core.MediaType;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This class contains several utility function which could be used in different
|
* This class contains several utility function which could be used in different
|
||||||
@ -182,4 +190,26 @@ public static PrintStream createPrintStream(String localDir, String nodeId,
|
|||||||
}
|
}
|
||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Redirect the {@link ContainerLogsRequest} to the NodeManager's
|
||||||
|
* NMWebServices.
|
||||||
|
*
|
||||||
|
* @param conf Configuration object
|
||||||
|
* @param webServiceClient client
|
||||||
|
* @param request the request for container logs
|
||||||
|
* @param logFile name of the log file
|
||||||
|
* @return response from NMWebServices
|
||||||
|
*/
|
||||||
|
public static ClientResponse getResponseFromNMWebService(Configuration conf,
|
||||||
|
Client webServiceClient, ContainerLogsRequest request, String logFile) {
|
||||||
|
WebResource webResource =
|
||||||
|
webServiceClient.resource(WebAppUtils.getHttpSchemePrefix(conf)
|
||||||
|
+ request.getNodeHttpAddress());
|
||||||
|
return webResource.path("ws").path("v1").path("node")
|
||||||
|
.path("containers").path(request.getContainerId()).path("logs")
|
||||||
|
.path(logFile)
|
||||||
|
.queryParam("size", Long.toString(request.getBytes()))
|
||||||
|
.accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -40,6 +40,7 @@
|
|||||||
import org.apache.hadoop.yarn.api.ApplicationConstants;
|
import org.apache.hadoop.yarn.api.ApplicationConstants;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
|
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
|
||||||
|
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
|
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -301,4 +302,17 @@ public static boolean shouldCountTowardsNodeBlacklisting(int exitStatus) {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns whether a given application state is final: FINISHED,
|
||||||
|
* FAILED or KILLED.
|
||||||
|
*
|
||||||
|
* @param appState application state
|
||||||
|
* @return whether the appState is final.
|
||||||
|
*/
|
||||||
|
public static boolean isApplicationFinalState(YarnApplicationState appState) {
|
||||||
|
return appState == YarnApplicationState.FINISHED
|
||||||
|
|| appState == YarnApplicationState.FAILED
|
||||||
|
|| appState == YarnApplicationState.KILLED;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -64,7 +64,7 @@
|
|||||||
import org.apache.hadoop.yarn.server.timeline.NameValuePair;
|
import org.apache.hadoop.yarn.server.timeline.NameValuePair;
|
||||||
import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
|
import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
|
||||||
import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field;
|
import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field;
|
||||||
import org.apache.hadoop.yarn.util.ConverterUtils;
|
import org.apache.hadoop.yarn.util.Apps;
|
||||||
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
||||||
|
|
||||||
import com.google.common.annotations.VisibleForTesting;
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
@ -412,7 +412,7 @@ private static ApplicationReportExt convertToApplicationReport(
|
|||||||
}
|
}
|
||||||
if (eventInfo.containsKey(
|
if (eventInfo.containsKey(
|
||||||
ApplicationMetricsConstants.STATE_EVENT_INFO)) {
|
ApplicationMetricsConstants.STATE_EVENT_INFO)) {
|
||||||
if (!isFinalState(state)) {
|
if (!Apps.isApplicationFinalState(state)) {
|
||||||
state = YarnApplicationState.valueOf(eventInfo.get(
|
state = YarnApplicationState.valueOf(eventInfo.get(
|
||||||
ApplicationMetricsConstants.STATE_EVENT_INFO).toString());
|
ApplicationMetricsConstants.STATE_EVENT_INFO).toString());
|
||||||
}
|
}
|
||||||
@ -475,12 +475,6 @@ private static long parseLong(Map<String, Object> entityInfo,
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static boolean isFinalState(YarnApplicationState state) {
|
|
||||||
return state == YarnApplicationState.FINISHED
|
|
||||||
|| state == YarnApplicationState.FAILED
|
|
||||||
|| state == YarnApplicationState.KILLED;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static ApplicationAttemptReport convertToApplicationAttemptReport(
|
private static ApplicationAttemptReport convertToApplicationAttemptReport(
|
||||||
TimelineEntity entity) {
|
TimelineEntity entity) {
|
||||||
String host = null;
|
String host = null;
|
||||||
|
@ -176,7 +176,7 @@ public Collection<ApplicationAttemptReport> run()
|
|||||||
&& conf.getBoolean(YarnConfiguration.RM_WEBAPP_UI_ACTIONS_ENABLED,
|
&& conf.getBoolean(YarnConfiguration.RM_WEBAPP_UI_ACTIONS_ENABLED,
|
||||||
YarnConfiguration.DEFAULT_RM_WEBAPP_UI_ACTIONS_ENABLED)
|
YarnConfiguration.DEFAULT_RM_WEBAPP_UI_ACTIONS_ENABLED)
|
||||||
&& !unsecuredUIForSecuredCluster
|
&& !unsecuredUIForSecuredCluster
|
||||||
&& !isAppInFinalState(app)) {
|
&& !Apps.isApplicationFinalState(app.getAppState())) {
|
||||||
// Application Kill
|
// Application Kill
|
||||||
html.div()
|
html.div()
|
||||||
.button()
|
.button()
|
||||||
@ -253,11 +253,9 @@ private void generateOverviewTable(AppInfo app, String schedulerPath,
|
|||||||
|| app.getTrackingUrl().equals(UNAVAILABLE) ? null : root_url(app
|
|| app.getTrackingUrl().equals(UNAVAILABLE) ? null : root_url(app
|
||||||
.getTrackingUrl()),
|
.getTrackingUrl()),
|
||||||
app.getTrackingUrl() == null
|
app.getTrackingUrl() == null
|
||||||
|| app.getTrackingUrl().equals(UNAVAILABLE) ? "Unassigned" : app
|
|| app.getTrackingUrl().equals(UNAVAILABLE) ? "Unassigned" :
|
||||||
.getAppState() == YarnApplicationState.FINISHED
|
Apps.isApplicationFinalState(app.getAppState()) ?
|
||||||
|| app.getAppState() == YarnApplicationState.FAILED
|
"History" : "ApplicationMaster");
|
||||||
|| app.getAppState() == YarnApplicationState.KILLED ? "History"
|
|
||||||
: "ApplicationMaster");
|
|
||||||
if (webUiType != null
|
if (webUiType != null
|
||||||
&& webUiType.equals(YarnWebParams.RM_WEB_UI)) {
|
&& webUiType.equals(YarnWebParams.RM_WEB_UI)) {
|
||||||
LogAggregationStatus status = getLogAggregationStatus();
|
LogAggregationStatus status = getLogAggregationStatus();
|
||||||
@ -448,10 +446,4 @@ public static String getCSRFHeaderString(Configuration conf) {
|
|||||||
}
|
}
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean isAppInFinalState(AppInfo app) {
|
|
||||||
return app.getAppState() == YarnApplicationState.FINISHED
|
|
||||||
|| app.getAppState() == YarnApplicationState.FAILED
|
|
||||||
|| app.getAppState() == YarnApplicationState.KILLED;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
@ -42,6 +42,7 @@
|
|||||||
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
|
import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
|
||||||
|
import org.apache.hadoop.yarn.util.Apps;
|
||||||
import org.apache.hadoop.yarn.webapp.BadRequestException;
|
import org.apache.hadoop.yarn.webapp.BadRequestException;
|
||||||
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
|
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
|
||||||
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE;
|
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE;
|
||||||
@ -211,10 +212,8 @@ protected void renderData(Block html) {
|
|||||||
|
|
||||||
String trackingUI =
|
String trackingUI =
|
||||||
app.getTrackingUrl() == null || app.getTrackingUrl().equals(UNAVAILABLE)
|
app.getTrackingUrl() == null || app.getTrackingUrl().equals(UNAVAILABLE)
|
||||||
? "Unassigned"
|
? "Unassigned" :
|
||||||
: app.getAppState() == YarnApplicationState.FINISHED
|
Apps.isApplicationFinalState(app.getAppState())
|
||||||
|| app.getAppState() == YarnApplicationState.FAILED
|
|
||||||
|| app.getAppState() == YarnApplicationState.KILLED
|
|
||||||
? "History" : "ApplicationMaster";
|
? "History" : "ApplicationMaster";
|
||||||
appsTableData.append(trackingURL == null ? "#" : "href='" + trackingURL)
|
appsTableData.append(trackingURL == null ? "#" : "href='" + trackingURL)
|
||||||
.append("'>").append(trackingUI).append("</a>\"],\n");
|
.append("'>").append(trackingUI).append("</a>\"],\n");
|
||||||
|
@ -27,6 +27,7 @@
|
|||||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.api.records.ContainerId;
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory;
|
import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory;
|
||||||
|
import org.apache.hadoop.yarn.util.Apps;
|
||||||
import org.apache.hadoop.yarn.webapp.BadRequestException;
|
import org.apache.hadoop.yarn.webapp.BadRequestException;
|
||||||
import org.apache.hadoop.yarn.webapp.NotFoundException;
|
import org.apache.hadoop.yarn.webapp.NotFoundException;
|
||||||
import org.codehaus.jettison.json.JSONException;
|
import org.codehaus.jettison.json.JSONException;
|
||||||
@ -97,7 +98,7 @@ public Response getContainerLogsInfo(HttpServletRequest req,
|
|||||||
}
|
}
|
||||||
// if the application finishes, directly find logs
|
// if the application finishes, directly find logs
|
||||||
// from HDFS.
|
// from HDFS.
|
||||||
if (LogWebServiceUtils.isFinishedState(appInfo.getAppState())) {
|
if (Apps.isApplicationFinalState(appInfo.getAppState())) {
|
||||||
return LogWebServiceUtils
|
return LogWebServiceUtils
|
||||||
.getContainerLogMeta(factory, appId, null, null, containerIdStr,
|
.getContainerLogMeta(factory, appId, null, null, containerIdStr,
|
||||||
false);
|
false);
|
||||||
@ -197,7 +198,7 @@ public Response getLogFile(HttpServletRequest req, String containerIdStr,
|
|||||||
filename, format, length, false);
|
filename, format, length, false);
|
||||||
}
|
}
|
||||||
String appOwner = appInfo.getUser();
|
String appOwner = appInfo.getUser();
|
||||||
if (LogWebServiceUtils.isFinishedState(appInfo.getAppState())) {
|
if (Apps.isApplicationFinalState(appInfo.getAppState())) {
|
||||||
// directly find logs from HDFS.
|
// directly find logs from HDFS.
|
||||||
return LogWebServiceUtils
|
return LogWebServiceUtils
|
||||||
.sendStreamOutputResponse(factory, appId, appOwner, null,
|
.sendStreamOutputResponse(factory, appId, appOwner, null,
|
||||||
|
@ -222,12 +222,6 @@ public static boolean isRunningState(YarnApplicationState appState) {
|
|||||||
return appState == YarnApplicationState.RUNNING;
|
return appState == YarnApplicationState.RUNNING;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static boolean isFinishedState(YarnApplicationState appState) {
|
|
||||||
return appState == YarnApplicationState.FINISHED
|
|
||||||
|| appState == YarnApplicationState.FAILED
|
|
||||||
|| appState == YarnApplicationState.KILLED;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected static UserGroupInformation getUser(HttpServletRequest req) {
|
protected static UserGroupInformation getUser(HttpServletRequest req) {
|
||||||
String remoteUser = req.getRemoteUser();
|
String remoteUser = req.getRemoteUser();
|
||||||
UserGroupInformation callerUGI = null;
|
UserGroupInformation callerUGI = null;
|
||||||
|
@ -38,6 +38,7 @@
|
|||||||
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
|
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
|
||||||
import org.apache.hadoop.yarn.server.webapp.AppsBlock;
|
import org.apache.hadoop.yarn.server.webapp.AppsBlock;
|
||||||
import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
|
import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
|
||||||
|
import org.apache.hadoop.yarn.util.Apps;
|
||||||
import org.apache.hadoop.yarn.webapp.View;
|
import org.apache.hadoop.yarn.webapp.View;
|
||||||
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
|
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
|
||||||
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE;
|
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE;
|
||||||
@ -197,10 +198,8 @@ protected void renderData(Block html) {
|
|||||||
app.getTrackingUrl() == null
|
app.getTrackingUrl() == null
|
||||||
|| app.getTrackingUrl().equals(UNAVAILABLE)
|
|| app.getTrackingUrl().equals(UNAVAILABLE)
|
||||||
|| app.getAppState() == YarnApplicationState.NEW ? "Unassigned"
|
|| app.getAppState() == YarnApplicationState.NEW ? "Unassigned"
|
||||||
: app.getAppState() == YarnApplicationState.FINISHED
|
: Apps.isApplicationFinalState(app.getAppState()) ?
|
||||||
|| app.getAppState() == YarnApplicationState.FAILED
|
"History" : "ApplicationMaster";
|
||||||
|| app.getAppState() == YarnApplicationState.KILLED ? "History"
|
|
||||||
: "ApplicationMaster";
|
|
||||||
appsTableData.append(trackingURL == null ? "#" : "href='" + trackingURL)
|
appsTableData.append(trackingURL == null ? "#" : "href='" + trackingURL)
|
||||||
.append("'>").append(trackingUI).append("</a>\",").append("\"")
|
.append("'>").append(trackingUI).append("</a>\",").append("\"")
|
||||||
.append(blacklistedNodesCount).append("\"],\n");
|
.append(blacklistedNodesCount).append("\"],\n");
|
||||||
|
@ -42,7 +42,6 @@
|
|||||||
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationReport;
|
import org.apache.hadoop.yarn.api.records.ApplicationReport;
|
||||||
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
|
||||||
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
|
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
|
||||||
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains;
|
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains;
|
||||||
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
|
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
|
||||||
@ -56,6 +55,7 @@
|
|||||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
import org.apache.hadoop.yarn.server.timeline.TimelineDataManager.CheckAcl;
|
import org.apache.hadoop.yarn.server.timeline.TimelineDataManager.CheckAcl;
|
||||||
import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager;
|
import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager;
|
||||||
|
import org.apache.hadoop.yarn.util.Apps;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
@ -104,11 +104,6 @@ public class EntityGroupFSTimelineStore extends CompositeService
|
|||||||
private static final FsPermission DONE_DIR_PERMISSION =
|
private static final FsPermission DONE_DIR_PERMISSION =
|
||||||
new FsPermission((short) 0700);
|
new FsPermission((short) 0700);
|
||||||
|
|
||||||
private static final EnumSet<YarnApplicationState>
|
|
||||||
APP_FINAL_STATES = EnumSet.of(
|
|
||||||
YarnApplicationState.FAILED,
|
|
||||||
YarnApplicationState.KILLED,
|
|
||||||
YarnApplicationState.FINISHED);
|
|
||||||
// Active dir: <activeRoot>/appId/attemptId/cacheId.log
|
// Active dir: <activeRoot>/appId/attemptId/cacheId.log
|
||||||
// Done dir: <doneRoot>/cluster_ts/hash1/hash2/appId/attemptId/cacheId.log
|
// Done dir: <doneRoot>/cluster_ts/hash1/hash2/appId/attemptId/cacheId.log
|
||||||
private static final String APP_DONE_DIR_PREFIX_FORMAT =
|
private static final String APP_DONE_DIR_PREFIX_FORMAT =
|
||||||
@ -649,8 +644,7 @@ private static synchronized AppState getAppState(ApplicationId appId,
|
|||||||
AppState appState = AppState.ACTIVE;
|
AppState appState = AppState.ACTIVE;
|
||||||
try {
|
try {
|
||||||
ApplicationReport report = yarnClient.getApplicationReport(appId);
|
ApplicationReport report = yarnClient.getApplicationReport(appId);
|
||||||
YarnApplicationState yarnState = report.getYarnApplicationState();
|
if (Apps.isApplicationFinalState(report.getYarnApplicationState())) {
|
||||||
if (APP_FINAL_STATES.contains(yarnState)) {
|
|
||||||
appState = AppState.COMPLETED;
|
appState = AppState.COMPLETED;
|
||||||
}
|
}
|
||||||
} catch (ApplicationNotFoundException e) {
|
} catch (ApplicationNotFoundException e) {
|
||||||
|
Loading…
Reference in New Issue
Block a user