From 15098df744deb045c0a33b095d45728d5b2af454 Mon Sep 17 00:00:00 2001 From: bibinchundatt Date: Mon, 4 Mar 2019 16:57:31 +0530 Subject: [PATCH] Revert "YARN-8132. Final Status of applications shown as UNDEFINED in ATS app queries. Contributed by Prabhu Joseph." This reverts commit a63c358b78ad1d4765d7f0b2d595c49b693efb3f. --- .../rmapp/attempt/RMAppAttemptImpl.java | 12 +++--- .../rmapp/TestRMAppTransitions.java | 43 ++++++++----------- .../attempt/TestRMAppAttemptTransitions.java | 3 +- 3 files changed, 24 insertions(+), 34 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 7b04ae7e72..03039daae9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -1372,7 +1372,7 @@ private void rememberTargetTransitionsAndStoreState(RMAppAttemptEvent event, setTrackingUrlToRMAppPage(stateToBeStored); } String finalTrackingUrl = getOriginalTrackingUrl(); - FinalApplicationStatus status = null; + FinalApplicationStatus finalStatus = null; int exitStatus = ContainerExitStatus.INVALID; switch (event.getType()) { case LAUNCH_FAILED: @@ -1387,7 +1387,7 @@ private void rememberTargetTransitionsAndStoreState(RMAppAttemptEvent event, diags.append(unregisterEvent.getDiagnosticMsg()); // reset finalTrackingUrl to url sent by am finalTrackingUrl = sanitizeTrackingUrl(unregisterEvent.getFinalTrackingUrl()); - status = unregisterEvent.getFinalApplicationStatus(); + finalStatus = unregisterEvent.getFinalApplicationStatus(); break; case CONTAINER_FINISHED: RMAppAttemptContainerFinishedEvent finishEvent = @@ -1396,10 +1396,8 @@ private void rememberTargetTransitionsAndStoreState(RMAppAttemptEvent event, exitStatus = finishEvent.getContainerStatus().getExitStatus(); break; case KILL: - status = FinalApplicationStatus.KILLED; break; case FAIL: - status = FinalApplicationStatus.FAILED; diags.append(event.getDiagnosticMsg()); break; case EXPIRE: @@ -1416,14 +1414,13 @@ private void rememberTargetTransitionsAndStoreState(RMAppAttemptEvent event, ApplicationAttemptStateData attemptState = ApplicationAttemptStateData .newInstance(applicationAttemptId, getMasterContainer(), rmStore.getCredentialsFromAppAttempt(this), startTime, - stateToBeStored, finalTrackingUrl, diags.toString(), status, - exitStatus, getFinishTime(), resUsage.getResourceUsageSecondsMap(), + stateToBeStored, finalTrackingUrl, diags.toString(), finalStatus, exitStatus, + getFinishTime(), resUsage.getResourceUsageSecondsMap(), this.attemptMetrics.getPreemptedResourceSecondsMap()); LOG.info("Updating application attempt " + applicationAttemptId + " with final state: " + targetedFinalState + ", and exit status: " + exitStatus); rmStore.updateApplicationAttemptState(attemptState); - finalStatus = status; } private static class FinalSavingTransition extends BaseTransition { @@ -1919,6 +1916,7 @@ private void updateInfoOnAMUnregister(RMAppAttemptEvent event) { (RMAppAttemptUnregistrationEvent) event; this.diagnostics.append(unregisterEvent.getDiagnosticMsg()); originalTrackingUrl = sanitizeTrackingUrl(unregisterEvent.getFinalTrackingUrl()); + finalStatus = unregisterEvent.getFinalApplicationStatus(); } private static final class ContainerFinishedTransition diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index 62a6706e94..32dcd3bebf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -722,7 +722,7 @@ public void testAppNewKill() throws IOException { sendAppUpdateSavedEvent(application); assertKilled(application); assertAppFinalStateNotSaved(application); - verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED); + verifyApplicationFinished(RMAppState.KILLED); verifyAppRemovedSchedulerEvent(application, RMAppState.KILLED); verifyRMAppFieldsForFinalTransitions(application); } @@ -741,7 +741,7 @@ public void testAppNewReject() throws IOException { sendAppUpdateSavedEvent(application); assertFailed(application, rejectedText); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED); + verifyApplicationFinished(RMAppState.FAILED); verifyRMAppFieldsForFinalTransitions(application); } @@ -759,7 +759,7 @@ public void testAppNewRejectAddToStore() throws IOException { sendAppUpdateSavedEvent(application); assertFailed(application, rejectedText); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED); + verifyApplicationFinished(RMAppState.FAILED); verifyRMAppFieldsForFinalTransitions(application); rmContext.getStateStore().removeApplication(application); } @@ -781,7 +781,7 @@ public void testAppNewSavingKill() throws IOException { rmDispatcher.await(); sendAppUpdateSavedEvent(application); assertKilled(application); - verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED); + verifyApplicationFinished(RMAppState.KILLED); verifyAppRemovedSchedulerEvent(application, RMAppState.KILLED); verifyRMAppFieldsForFinalTransitions(application); } @@ -800,7 +800,7 @@ public void testAppNewSavingReject() throws IOException { sendAppUpdateSavedEvent(application); assertFailed(application, rejectedText); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED); + verifyApplicationFinished(RMAppState.FAILED); verifyRMAppFieldsForFinalTransitions(application); } @@ -817,7 +817,7 @@ public void testAppNewSavingSaveReject() throws IOException { assertFailed(application, rejectedText); verify(store, times(0)).updateApplicationState( any(ApplicationStateData.class)); - verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED); + verifyApplicationFinished(RMAppState.FAILED); assertTimesAtFinish(application); } @@ -835,7 +835,7 @@ public void testAppSubmittedRejected() throws IOException { sendAppUpdateSavedEvent(application); assertFailed(application, rejectedText); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED); + verifyApplicationFinished(RMAppState.FAILED); verifyRMAppFieldsForFinalTransitions(application); } @@ -857,7 +857,7 @@ public void testAppSubmittedKill() throws IOException, InterruptedException { sendAppUpdateSavedEvent(application); assertKilled(application); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED); + verifyApplicationFinished(RMAppState.KILLED); verifyAppRemovedSchedulerEvent(application, RMAppState.KILLED); verifyRMAppFieldsForFinalTransitions(application); } @@ -894,7 +894,7 @@ public void testAppAcceptedFailed() throws IOException { sendAppUpdateSavedEvent(application); assertFailed(application, ".*" + message + ".*Failing the application.*"); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED); + verifyApplicationFinished(RMAppState.FAILED); } @Test @@ -921,7 +921,7 @@ public void testAppAcceptedKill() throws IOException, InterruptedException { sendAppUpdateSavedEvent(application); assertKilled(application); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED); + verifyApplicationFinished(RMAppState.KILLED); verifyAppRemovedSchedulerEvent(application, RMAppState.KILLED); verifyRMAppFieldsForFinalTransitions(application); } @@ -945,7 +945,7 @@ public void testAppAcceptedAttemptKilled() throws IOException, sendAppUpdateSavedEvent(application); assertKilled(application); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED); + verifyApplicationFinished(RMAppState.KILLED); verifyAppRemovedSchedulerEvent(application, RMAppState.KILLED); } @@ -969,7 +969,7 @@ public void testAppRunningKill() throws IOException { sendAttemptUpdateSavedEvent(application); sendAppUpdateSavedEvent(application); assertKilled(application); - verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED); + verifyApplicationFinished(RMAppState.KILLED); verifyAppRemovedSchedulerEvent(application, RMAppState.KILLED); verifyRMAppFieldsForFinalTransitions(application); } @@ -1028,7 +1028,7 @@ public void testAppRunningFailed() throws IOException { rmDispatcher.await(); assertFailed(application, ".*Failing the application.*"); assertAppFinalStateSaved(application); - verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED); + verifyApplicationFinished(RMAppState.FAILED); verifyRMAppFieldsForFinalTransitions(application); } @@ -1090,9 +1090,7 @@ public void testAppFinishedFinished() throws Exception { StringBuilder diag = application.getDiagnostics(); Assert.assertEquals("application diagnostics is not correct", "", diag.toString()); - // finished without a proper final state is the same as failed - verifyApplicationFinished(RMAppState.FINISHED, - FinalApplicationStatus.FAILED); + verifyApplicationFinished(RMAppState.FINISHED); verifyAppRemovedSchedulerEvent(application, RMAppState.FINISHED); verifyRMAppFieldsForFinalTransitions(application); } @@ -1120,7 +1118,7 @@ public void testAppFailedFailed() throws IOException { rmDispatcher.await(); assertTimesAtFinish(application); assertAppState(RMAppState.FAILED, application); - verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED); + verifyApplicationFinished(RMAppState.FAILED); assertTimesAtFinish(application); assertAppState(RMAppState.FAILED, application); @@ -1175,7 +1173,7 @@ public void testAppKilledKilled() throws IOException { rmDispatcher.await(); assertTimesAtFinish(application); assertAppState(RMAppState.KILLED, application); - verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED); + verifyApplicationFinished(RMAppState.KILLED); assertTimesAtFinish(application); assertAppState(RMAppState.KILLED, application); @@ -1311,20 +1309,15 @@ private void verifyAppCompletedEvent(RMApp app) { rmAppManagerEvent.getApplicationId().getId()); } - private void verifyApplicationFinished(RMAppState state, - FinalApplicationStatus finalAppStatus) { + private void verifyApplicationFinished(RMAppState state) { ArgumentCaptor finalState = ArgumentCaptor.forClass(RMAppState.class); verify(writer).applicationFinished(any(RMApp.class), finalState.capture()); Assert.assertEquals(state, finalState.getValue()); finalState = ArgumentCaptor.forClass(RMAppState.class); - ArgumentCaptor app = - ArgumentCaptor.forClass(RMApp.class); - verify(publisher).appFinished(app.capture(), finalState.capture(), + verify(publisher).appFinished(any(RMApp.class), finalState.capture(), anyLong()); Assert.assertEquals(state, finalState.getValue()); - Assert.assertEquals(finalAppStatus, - app.getValue().getFinalApplicationStatus()); } private void verifyAppRemovedSchedulerEvent(RMApp app, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index 613bd92ca8..b5c15cd840 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -436,8 +436,7 @@ private void testAppAttemptKilledState(Container amContainer, assertEquals(amContainer, applicationAttempt.getMasterContainer()); assertEquals(0.0, (double)applicationAttempt.getProgress(), 0.0001); assertEquals(0, application.getRanNodes().size()); - assertEquals(FinalApplicationStatus.KILLED, - applicationAttempt.getFinalApplicationStatus()); + assertNull(applicationAttempt.getFinalApplicationStatus()); verifyTokenCount(applicationAttempt.getAppAttemptId(), 1); verifyAttemptFinalStateSaved(); assertFalse(transferStateFromPreviousAttempt);