big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-8132. Final Status of applications shown as UNDEFINED in ATS app queries. Contributed by Prabhu Joseph.

Browse Source
This commit is contained in:
bibinchundatt 2019-02-21 13:29:01 +05:30
parent a87e458432
commit a63c358b78
3 changed files with 34 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
View File

@ -1372,7 +1372,7 @@ private void rememberTargetTransitionsAndStoreState(RMAppAttemptEvent event,
setTrackingUrlToRMAppPage(stateToBeStored); setTrackingUrlToRMAppPage(stateToBeStored);
} }
String finalTrackingUrl = getOriginalTrackingUrl(); String finalTrackingUrl = getOriginalTrackingUrl();
FinalApplicationStatus finalStatus = null; FinalApplicationStatus status = null;
int exitStatus = ContainerExitStatus.INVALID; int exitStatus = ContainerExitStatus.INVALID;
switch (event.getType()) { switch (event.getType()) {
case LAUNCH_FAILED: case LAUNCH_FAILED:
@ -1387,7 +1387,7 @@ private void rememberTargetTransitionsAndStoreState(RMAppAttemptEvent event,
diags.append(unregisterEvent.getDiagnosticMsg()); diags.append(unregisterEvent.getDiagnosticMsg());
// reset finalTrackingUrl to url sent by am // reset finalTrackingUrl to url sent by am
finalTrackingUrl = sanitizeTrackingUrl(unregisterEvent.getFinalTrackingUrl()); finalTrackingUrl = sanitizeTrackingUrl(unregisterEvent.getFinalTrackingUrl());
finalStatus = unregisterEvent.getFinalApplicationStatus(); status = unregisterEvent.getFinalApplicationStatus();
break; break;
case CONTAINER_FINISHED: case CONTAINER_FINISHED:
RMAppAttemptContainerFinishedEvent finishEvent = RMAppAttemptContainerFinishedEvent finishEvent =
@ -1396,8 +1396,10 @@ private void rememberTargetTransitionsAndStoreState(RMAppAttemptEvent event,
exitStatus = finishEvent.getContainerStatus().getExitStatus(); exitStatus = finishEvent.getContainerStatus().getExitStatus();
break; break;
case KILL: case KILL:
status = FinalApplicationStatus.KILLED;
break; break;
case FAIL: case FAIL:
status = FinalApplicationStatus.FAILED;
diags.append(event.getDiagnosticMsg()); diags.append(event.getDiagnosticMsg());
break; break;
case EXPIRE: case EXPIRE:
@ -1414,13 +1416,14 @@ private void rememberTargetTransitionsAndStoreState(RMAppAttemptEvent event,
ApplicationAttemptStateData attemptState = ApplicationAttemptStateData ApplicationAttemptStateData attemptState = ApplicationAttemptStateData
.newInstance(applicationAttemptId, getMasterContainer(), .newInstance(applicationAttemptId, getMasterContainer(),
rmStore.getCredentialsFromAppAttempt(this), startTime, rmStore.getCredentialsFromAppAttempt(this), startTime,
stateToBeStored, finalTrackingUrl, diags.toString(), finalStatus, exitStatus, stateToBeStored, finalTrackingUrl, diags.toString(), status,
getFinishTime(), resUsage.getResourceUsageSecondsMap(), exitStatus, getFinishTime(), resUsage.getResourceUsageSecondsMap(),
this.attemptMetrics.getPreemptedResourceSecondsMap()); this.attemptMetrics.getPreemptedResourceSecondsMap());
LOG.info("Updating application attempt " + applicationAttemptId LOG.info("Updating application attempt " + applicationAttemptId
+ " with final state: " + targetedFinalState + ", and exit status: " + " with final state: " + targetedFinalState + ", and exit status: "
+ exitStatus); + exitStatus);
rmStore.updateApplicationAttemptState(attemptState); rmStore.updateApplicationAttemptState(attemptState);
finalStatus = status;
} }
private static class FinalSavingTransition extends BaseTransition { private static class FinalSavingTransition extends BaseTransition {
@ -1916,7 +1919,6 @@ private void updateInfoOnAMUnregister(RMAppAttemptEvent event) {
(RMAppAttemptUnregistrationEvent) event; (RMAppAttemptUnregistrationEvent) event;
this.diagnostics.append(unregisterEvent.getDiagnosticMsg()); this.diagnostics.append(unregisterEvent.getDiagnosticMsg());
originalTrackingUrl = sanitizeTrackingUrl(unregisterEvent.getFinalTrackingUrl()); originalTrackingUrl = sanitizeTrackingUrl(unregisterEvent.getFinalTrackingUrl());
finalStatus = unregisterEvent.getFinalApplicationStatus();
} }
private static final class ContainerFinishedTransition private static final class ContainerFinishedTransition

43
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
View File

@ -722,7 +722,7 @@ public void testAppNewKill() throws IOException {
sendAppUpdateSavedEvent(application); sendAppUpdateSavedEvent(application);
assertKilled(application); assertKilled(application);
assertAppFinalStateNotSaved(application); assertAppFinalStateNotSaved(application);
verifyApplicationFinished(RMAppState.KILLED); verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED);
verifyAppRemovedSchedulerEvent(application, RMAppState.KILLED); verifyAppRemovedSchedulerEvent(application, RMAppState.KILLED);
verifyRMAppFieldsForFinalTransitions(application); verifyRMAppFieldsForFinalTransitions(application);
} }
@ -741,7 +741,7 @@ public void testAppNewReject() throws IOException {
sendAppUpdateSavedEvent(application); sendAppUpdateSavedEvent(application);
assertFailed(application, rejectedText); assertFailed(application, rejectedText);
assertAppFinalStateSaved(application); assertAppFinalStateSaved(application);
verifyApplicationFinished(RMAppState.FAILED); verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED);
verifyRMAppFieldsForFinalTransitions(application); verifyRMAppFieldsForFinalTransitions(application);
} }
@ -759,7 +759,7 @@ public void testAppNewRejectAddToStore() throws IOException {
sendAppUpdateSavedEvent(application); sendAppUpdateSavedEvent(application);
assertFailed(application, rejectedText); assertFailed(application, rejectedText);
assertAppFinalStateSaved(application); assertAppFinalStateSaved(application);
verifyApplicationFinished(RMAppState.FAILED); verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED);
verifyRMAppFieldsForFinalTransitions(application); verifyRMAppFieldsForFinalTransitions(application);
rmContext.getStateStore().removeApplication(application); rmContext.getStateStore().removeApplication(application);
} }
@ -781,7 +781,7 @@ public void testAppNewSavingKill() throws IOException {
rmDispatcher.await(); rmDispatcher.await();
sendAppUpdateSavedEvent(application); sendAppUpdateSavedEvent(application);
assertKilled(application); assertKilled(application);
verifyApplicationFinished(RMAppState.KILLED); verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED);
verifyAppRemovedSchedulerEvent(application, RMAppState.KILLED); verifyAppRemovedSchedulerEvent(application, RMAppState.KILLED);
verifyRMAppFieldsForFinalTransitions(application); verifyRMAppFieldsForFinalTransitions(application);
} }
@ -800,7 +800,7 @@ public void testAppNewSavingReject() throws IOException {
sendAppUpdateSavedEvent(application); sendAppUpdateSavedEvent(application);
assertFailed(application, rejectedText); assertFailed(application, rejectedText);
assertAppFinalStateSaved(application); assertAppFinalStateSaved(application);
verifyApplicationFinished(RMAppState.FAILED); verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED);
verifyRMAppFieldsForFinalTransitions(application); verifyRMAppFieldsForFinalTransitions(application);
} }
@ -817,7 +817,7 @@ public void testAppNewSavingSaveReject() throws IOException {
assertFailed(application, rejectedText); assertFailed(application, rejectedText);
verify(store, times(0)).updateApplicationState( verify(store, times(0)).updateApplicationState(
any(ApplicationStateData.class)); any(ApplicationStateData.class));
verifyApplicationFinished(RMAppState.FAILED); verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED);
assertTimesAtFinish(application); assertTimesAtFinish(application);
} }
@ -835,7 +835,7 @@ public void testAppSubmittedRejected() throws IOException {
sendAppUpdateSavedEvent(application); sendAppUpdateSavedEvent(application);
assertFailed(application, rejectedText); assertFailed(application, rejectedText);
assertAppFinalStateSaved(application); assertAppFinalStateSaved(application);
verifyApplicationFinished(RMAppState.FAILED); verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED);
verifyRMAppFieldsForFinalTransitions(application); verifyRMAppFieldsForFinalTransitions(application);
} }
@ -857,7 +857,7 @@ public void testAppSubmittedKill() throws IOException, InterruptedException {
sendAppUpdateSavedEvent(application); sendAppUpdateSavedEvent(application);
assertKilled(application); assertKilled(application);
assertAppFinalStateSaved(application); assertAppFinalStateSaved(application);
verifyApplicationFinished(RMAppState.KILLED); verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED);
verifyAppRemovedSchedulerEvent(application, RMAppState.KILLED); verifyAppRemovedSchedulerEvent(application, RMAppState.KILLED);
verifyRMAppFieldsForFinalTransitions(application); verifyRMAppFieldsForFinalTransitions(application);
} }
@ -894,7 +894,7 @@ public void testAppAcceptedFailed() throws IOException {
sendAppUpdateSavedEvent(application); sendAppUpdateSavedEvent(application);
assertFailed(application, ".*" + message + ".*Failing the application.*"); assertFailed(application, ".*" + message + ".*Failing the application.*");
assertAppFinalStateSaved(application); assertAppFinalStateSaved(application);
verifyApplicationFinished(RMAppState.FAILED); verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED);
} }
@Test @Test
@ -921,7 +921,7 @@ public void testAppAcceptedKill() throws IOException, InterruptedException {
sendAppUpdateSavedEvent(application); sendAppUpdateSavedEvent(application);
assertKilled(application); assertKilled(application);
assertAppFinalStateSaved(application); assertAppFinalStateSaved(application);
verifyApplicationFinished(RMAppState.KILLED); verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED);
verifyAppRemovedSchedulerEvent(application, RMAppState.KILLED); verifyAppRemovedSchedulerEvent(application, RMAppState.KILLED);
verifyRMAppFieldsForFinalTransitions(application); verifyRMAppFieldsForFinalTransitions(application);
} }
@ -945,7 +945,7 @@ public void testAppAcceptedAttemptKilled() throws IOException,
sendAppUpdateSavedEvent(application); sendAppUpdateSavedEvent(application);
assertKilled(application); assertKilled(application);
assertAppFinalStateSaved(application); assertAppFinalStateSaved(application);
verifyApplicationFinished(RMAppState.KILLED); verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED);
verifyAppRemovedSchedulerEvent(application, RMAppState.KILLED); verifyAppRemovedSchedulerEvent(application, RMAppState.KILLED);
} }
@ -969,7 +969,7 @@ public void testAppRunningKill() throws IOException {
sendAttemptUpdateSavedEvent(application); sendAttemptUpdateSavedEvent(application);
sendAppUpdateSavedEvent(application); sendAppUpdateSavedEvent(application);
assertKilled(application); assertKilled(application);
verifyApplicationFinished(RMAppState.KILLED); verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED);
verifyAppRemovedSchedulerEvent(application, RMAppState.KILLED); verifyAppRemovedSchedulerEvent(application, RMAppState.KILLED);
verifyRMAppFieldsForFinalTransitions(application); verifyRMAppFieldsForFinalTransitions(application);
} }
@ -1028,7 +1028,7 @@ public void testAppRunningFailed() throws IOException {
rmDispatcher.await(); rmDispatcher.await();
assertFailed(application, ".*Failing the application.*"); assertFailed(application, ".*Failing the application.*");
assertAppFinalStateSaved(application); assertAppFinalStateSaved(application);
verifyApplicationFinished(RMAppState.FAILED); verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED);
verifyRMAppFieldsForFinalTransitions(application); verifyRMAppFieldsForFinalTransitions(application);
} }
@ -1090,7 +1090,9 @@ public void testAppFinishedFinished() throws Exception {
StringBuilder diag = application.getDiagnostics(); StringBuilder diag = application.getDiagnostics();
Assert.assertEquals("application diagnostics is not correct", Assert.assertEquals("application diagnostics is not correct",
"", diag.toString()); "", diag.toString());
verifyApplicationFinished(RMAppState.FINISHED); // finished without a proper final state is the same as failed
verifyApplicationFinished(RMAppState.FINISHED,
FinalApplicationStatus.FAILED);
verifyAppRemovedSchedulerEvent(application, RMAppState.FINISHED); verifyAppRemovedSchedulerEvent(application, RMAppState.FINISHED);
verifyRMAppFieldsForFinalTransitions(application); verifyRMAppFieldsForFinalTransitions(application);
} }
@ -1118,7 +1120,7 @@ public void testAppFailedFailed() throws IOException {
rmDispatcher.await(); rmDispatcher.await();
assertTimesAtFinish(application); assertTimesAtFinish(application);
assertAppState(RMAppState.FAILED, application); assertAppState(RMAppState.FAILED, application);
verifyApplicationFinished(RMAppState.FAILED); verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED);
assertTimesAtFinish(application); assertTimesAtFinish(application);
assertAppState(RMAppState.FAILED, application); assertAppState(RMAppState.FAILED, application);
@ -1173,7 +1175,7 @@ public void testAppKilledKilled() throws IOException {
rmDispatcher.await(); rmDispatcher.await();
assertTimesAtFinish(application); assertTimesAtFinish(application);
assertAppState(RMAppState.KILLED, application); assertAppState(RMAppState.KILLED, application);
verifyApplicationFinished(RMAppState.KILLED); verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED);
assertTimesAtFinish(application); assertTimesAtFinish(application);
assertAppState(RMAppState.KILLED, application); assertAppState(RMAppState.KILLED, application);
@ -1309,15 +1311,20 @@ private void verifyAppCompletedEvent(RMApp app) {
rmAppManagerEvent.getApplicationId().getId()); rmAppManagerEvent.getApplicationId().getId());
} }
private void verifyApplicationFinished(RMAppState state) { private void verifyApplicationFinished(RMAppState state,
FinalApplicationStatus finalAppStatus) {
ArgumentCaptor<RMAppState> finalState = ArgumentCaptor<RMAppState> finalState =
ArgumentCaptor.forClass(RMAppState.class); ArgumentCaptor.forClass(RMAppState.class);
verify(writer).applicationFinished(any(RMApp.class), finalState.capture()); verify(writer).applicationFinished(any(RMApp.class), finalState.capture());
Assert.assertEquals(state, finalState.getValue()); Assert.assertEquals(state, finalState.getValue());
finalState = ArgumentCaptor.forClass(RMAppState.class); finalState = ArgumentCaptor.forClass(RMAppState.class);
verify(publisher).appFinished(any(RMApp.class), finalState.capture(), ArgumentCaptor<RMApp> app =
ArgumentCaptor.forClass(RMApp.class);
verify(publisher).appFinished(app.capture(), finalState.capture(),
anyLong()); anyLong());
Assert.assertEquals(state, finalState.getValue()); Assert.assertEquals(state, finalState.getValue());
Assert.assertEquals(finalAppStatus,
app.getValue().getFinalApplicationStatus());
} }
private void verifyAppRemovedSchedulerEvent(RMApp app, private void verifyAppRemovedSchedulerEvent(RMApp app,

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
View File

@ -436,7 +436,8 @@ private void testAppAttemptKilledState(Container amContainer,
assertEquals(amContainer, applicationAttempt.getMasterContainer()); assertEquals(amContainer, applicationAttempt.getMasterContainer());
assertEquals(0.0, (double)applicationAttempt.getProgress(), 0.0001); assertEquals(0.0, (double)applicationAttempt.getProgress(), 0.0001);
assertEquals(0, application.getRanNodes().size()); assertEquals(0, application.getRanNodes().size());
assertNull(applicationAttempt.getFinalApplicationStatus()); assertEquals(FinalApplicationStatus.KILLED,
applicationAttempt.getFinalApplicationStatus());
verifyTokenCount(applicationAttempt.getAppAttemptId(), 1); verifyTokenCount(applicationAttempt.getAppAttemptId(), 1);
verifyAttemptFinalStateSaved(); verifyAttemptFinalStateSaved();
assertFalse(transferStateFromPreviousAttempt); assertFalse(transferStateFromPreviousAttempt);