YARN-8132. Final Status of applications shown as UNDEFINED in ATS app queries. Contributed by Prabhu Joseph.

2019-02-21 13:29:01 +05:30 · 2019-02-21 13:29:01 +05:30 · a63c358b78
commit a63c358b78
parent a87e458432
3 changed files with 34 additions and 24 deletions
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@ -1372,7 +1372,7 @@ private void rememberTargetTransitionsAndStoreState(RMAppAttemptEvent event,
      setTrackingUrlToRMAppPage(stateToBeStored);
    }
    String finalTrackingUrl = getOriginalTrackingUrl();
-    FinalApplicationStatus finalStatus = null;
+    FinalApplicationStatus status = null;
    int exitStatus = ContainerExitStatus.INVALID;
    switch (event.getType()) {
    case LAUNCH_FAILED:
@ -1387,7 +1387,7 @@ private void rememberTargetTransitionsAndStoreState(RMAppAttemptEvent event,
      diags.append(unregisterEvent.getDiagnosticMsg());
      // reset finalTrackingUrl to url sent by am
      finalTrackingUrl = sanitizeTrackingUrl(unregisterEvent.getFinalTrackingUrl());
-      finalStatus = unregisterEvent.getFinalApplicationStatus();
+      status = unregisterEvent.getFinalApplicationStatus();
      break;
    case CONTAINER_FINISHED:
      RMAppAttemptContainerFinishedEvent finishEvent =
@ -1396,8 +1396,10 @@ private void rememberTargetTransitionsAndStoreState(RMAppAttemptEvent event,
      exitStatus = finishEvent.getContainerStatus().getExitStatus();
      break;
    case KILL:
+      status = FinalApplicationStatus.KILLED;
      break;
    case FAIL:
+      status = FinalApplicationStatus.FAILED;
      diags.append(event.getDiagnosticMsg());
      break;
    case EXPIRE:
@ -1414,13 +1416,14 @@ private void rememberTargetTransitionsAndStoreState(RMAppAttemptEvent event,
    ApplicationAttemptStateData attemptState = ApplicationAttemptStateData
        .newInstance(applicationAttemptId, getMasterContainer(),
            rmStore.getCredentialsFromAppAttempt(this), startTime,
-            stateToBeStored, finalTrackingUrl, diags.toString(), finalStatus, exitStatus,
-            getFinishTime(), resUsage.getResourceUsageSecondsMap(),
+            stateToBeStored, finalTrackingUrl, diags.toString(), status,
+            exitStatus, getFinishTime(), resUsage.getResourceUsageSecondsMap(),
            this.attemptMetrics.getPreemptedResourceSecondsMap());
    LOG.info("Updating application attempt " + applicationAttemptId
        + " with final state: " + targetedFinalState + ", and exit status: "
        + exitStatus);
    rmStore.updateApplicationAttemptState(attemptState);
+    finalStatus = status;
  }

  private static class FinalSavingTransition extends BaseTransition {
@ -1916,7 +1919,6 @@ private void updateInfoOnAMUnregister(RMAppAttemptEvent event) {
        (RMAppAttemptUnregistrationEvent) event;
    this.diagnostics.append(unregisterEvent.getDiagnosticMsg());
    originalTrackingUrl = sanitizeTrackingUrl(unregisterEvent.getFinalTrackingUrl());
-    finalStatus = unregisterEvent.getFinalApplicationStatus();
  }

  private static final class ContainerFinishedTransition
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
@ -722,7 +722,7 @@ public void testAppNewKill() throws IOException {
    sendAppUpdateSavedEvent(application);
    assertKilled(application);
    assertAppFinalStateNotSaved(application);
-    verifyApplicationFinished(RMAppState.KILLED);
+    verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED);
    verifyAppRemovedSchedulerEvent(application, RMAppState.KILLED);
    verifyRMAppFieldsForFinalTransitions(application);
  }
@ -741,7 +741,7 @@ public void testAppNewReject() throws IOException {
    sendAppUpdateSavedEvent(application);
    assertFailed(application, rejectedText);
    assertAppFinalStateSaved(application);
-    verifyApplicationFinished(RMAppState.FAILED);
+    verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED);
    verifyRMAppFieldsForFinalTransitions(application);
  }

@ -759,7 +759,7 @@ public void testAppNewRejectAddToStore() throws IOException {
    sendAppUpdateSavedEvent(application);
    assertFailed(application, rejectedText);
    assertAppFinalStateSaved(application);
-    verifyApplicationFinished(RMAppState.FAILED);
+    verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED);
    verifyRMAppFieldsForFinalTransitions(application);
    rmContext.getStateStore().removeApplication(application);
  }
@ -781,7 +781,7 @@ public void testAppNewSavingKill() throws IOException {
    rmDispatcher.await();
    sendAppUpdateSavedEvent(application);
    assertKilled(application);
-    verifyApplicationFinished(RMAppState.KILLED);
+    verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED);
    verifyAppRemovedSchedulerEvent(application, RMAppState.KILLED);
    verifyRMAppFieldsForFinalTransitions(application);
  }
@ -800,7 +800,7 @@ public void testAppNewSavingReject() throws IOException {
    sendAppUpdateSavedEvent(application);
    assertFailed(application, rejectedText);
    assertAppFinalStateSaved(application);
-    verifyApplicationFinished(RMAppState.FAILED);
+    verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED);
    verifyRMAppFieldsForFinalTransitions(application);
  }

@ -817,7 +817,7 @@ public void testAppNewSavingSaveReject() throws IOException {
    assertFailed(application, rejectedText);
    verify(store, times(0)).updateApplicationState(
        any(ApplicationStateData.class));
-    verifyApplicationFinished(RMAppState.FAILED);
+    verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED);
    assertTimesAtFinish(application);
  }

@ -835,7 +835,7 @@ public void testAppSubmittedRejected() throws IOException {
    sendAppUpdateSavedEvent(application);
    assertFailed(application, rejectedText);
    assertAppFinalStateSaved(application);
-    verifyApplicationFinished(RMAppState.FAILED);
+    verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED);
    verifyRMAppFieldsForFinalTransitions(application);
  }

@ -857,7 +857,7 @@ public void testAppSubmittedKill() throws IOException, InterruptedException {
    sendAppUpdateSavedEvent(application);
    assertKilled(application);
    assertAppFinalStateSaved(application);
-    verifyApplicationFinished(RMAppState.KILLED);
+    verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED);
    verifyAppRemovedSchedulerEvent(application, RMAppState.KILLED);
    verifyRMAppFieldsForFinalTransitions(application);
  }
@ -894,7 +894,7 @@ public void testAppAcceptedFailed() throws IOException {
    sendAppUpdateSavedEvent(application);
    assertFailed(application, ".*" + message + ".*Failing the application.*");
    assertAppFinalStateSaved(application);
-    verifyApplicationFinished(RMAppState.FAILED);
+    verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED);
  }

  @Test
@ -921,7 +921,7 @@ public void testAppAcceptedKill() throws IOException, InterruptedException {
    sendAppUpdateSavedEvent(application);
    assertKilled(application);
    assertAppFinalStateSaved(application);
-    verifyApplicationFinished(RMAppState.KILLED);
+    verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED);
    verifyAppRemovedSchedulerEvent(application, RMAppState.KILLED);
    verifyRMAppFieldsForFinalTransitions(application);
  }
@ -945,7 +945,7 @@ public void testAppAcceptedAttemptKilled() throws IOException,
    sendAppUpdateSavedEvent(application);
    assertKilled(application);
    assertAppFinalStateSaved(application);
-    verifyApplicationFinished(RMAppState.KILLED);
+    verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED);
    verifyAppRemovedSchedulerEvent(application, RMAppState.KILLED);
  }

@ -969,7 +969,7 @@ public void testAppRunningKill() throws IOException {
    sendAttemptUpdateSavedEvent(application);
    sendAppUpdateSavedEvent(application);
    assertKilled(application);
-    verifyApplicationFinished(RMAppState.KILLED);
+    verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED);
    verifyAppRemovedSchedulerEvent(application, RMAppState.KILLED);
    verifyRMAppFieldsForFinalTransitions(application);
  }
@ -1028,7 +1028,7 @@ public void testAppRunningFailed() throws IOException {
    rmDispatcher.await();
    assertFailed(application, ".*Failing the application.*");
    assertAppFinalStateSaved(application);
-    verifyApplicationFinished(RMAppState.FAILED);
+    verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED);
    verifyRMAppFieldsForFinalTransitions(application);
  }

@ -1090,7 +1090,9 @@ public void testAppFinishedFinished() throws Exception {
    StringBuilder diag = application.getDiagnostics();
    Assert.assertEquals("application diagnostics is not correct",
        "", diag.toString());
-    verifyApplicationFinished(RMAppState.FINISHED);
+    // finished without a proper final state is the same as failed
+    verifyApplicationFinished(RMAppState.FINISHED,
+        FinalApplicationStatus.FAILED);
    verifyAppRemovedSchedulerEvent(application, RMAppState.FINISHED);
    verifyRMAppFieldsForFinalTransitions(application);
  }
@ -1118,7 +1120,7 @@ public void testAppFailedFailed() throws IOException {
    rmDispatcher.await();
    assertTimesAtFinish(application);
    assertAppState(RMAppState.FAILED, application);
-    verifyApplicationFinished(RMAppState.FAILED);
+    verifyApplicationFinished(RMAppState.FAILED, FinalApplicationStatus.FAILED);

    assertTimesAtFinish(application);
    assertAppState(RMAppState.FAILED, application);
@ -1173,7 +1175,7 @@ public void testAppKilledKilled() throws IOException {
    rmDispatcher.await();
    assertTimesAtFinish(application);
    assertAppState(RMAppState.KILLED, application);
-    verifyApplicationFinished(RMAppState.KILLED);
+    verifyApplicationFinished(RMAppState.KILLED, FinalApplicationStatus.KILLED);

    assertTimesAtFinish(application);
    assertAppState(RMAppState.KILLED, application);
@ -1309,15 +1311,20 @@ private void verifyAppCompletedEvent(RMApp app) {
        rmAppManagerEvent.getApplicationId().getId());
  }

-  private void verifyApplicationFinished(RMAppState state) {
+  private void verifyApplicationFinished(RMAppState state,
+      FinalApplicationStatus finalAppStatus) {
    ArgumentCaptor<RMAppState> finalState =
        ArgumentCaptor.forClass(RMAppState.class);
    verify(writer).applicationFinished(any(RMApp.class), finalState.capture());
    Assert.assertEquals(state, finalState.getValue());
    finalState = ArgumentCaptor.forClass(RMAppState.class);
-    verify(publisher).appFinished(any(RMApp.class), finalState.capture(),
+    ArgumentCaptor<RMApp> app =
+        ArgumentCaptor.forClass(RMApp.class);
+    verify(publisher).appFinished(app.capture(), finalState.capture(),
        anyLong());
    Assert.assertEquals(state, finalState.getValue());
+    Assert.assertEquals(finalAppStatus,
+        app.getValue().getFinalApplicationStatus());
  }
  
  private void verifyAppRemovedSchedulerEvent(RMApp app,
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
@ -436,7 +436,8 @@ private void testAppAttemptKilledState(Container amContainer,
    assertEquals(amContainer, applicationAttempt.getMasterContainer());
    assertEquals(0.0, (double)applicationAttempt.getProgress(), 0.0001);
    assertEquals(0, application.getRanNodes().size());
-    assertNull(applicationAttempt.getFinalApplicationStatus());
+    assertEquals(FinalApplicationStatus.KILLED,
+        applicationAttempt.getFinalApplicationStatus());
    verifyTokenCount(applicationAttempt.getAppAttemptId(), 1);
    verifyAttemptFinalStateSaved();
    assertFalse(transferStateFromPreviousAttempt);