From 7f55a18071ebe1c5591ff8a410b227c855c3f428 Mon Sep 17 00:00:00 2001 From: Wangda Tan Date: Thu, 12 Nov 2015 11:23:40 -0800 Subject: [PATCH] YARN-4347. Resource manager fails with Null pointer exception. (Jian He via wangda) --- hadoop-yarn-project/CHANGES.txt | 2 + .../resourcemanager/rmapp/RMAppImpl.java | 2 +- .../rmapp/attempt/RMAppAttemptImpl.java | 26 ++++++++++-- .../TestWorkPreservingRMRestart.java | 41 +++++++++++++++++++ 4 files changed, 67 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 52c6669865..aaa222d6dc 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -1065,6 +1065,8 @@ Release 2.7.3 - UNRELEASED BUG FIXES + YARN-4347. Resource manager fails with Null pointer exception. (Jian He via wangda) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 41254d8124..bcfce9a89a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -1382,7 +1382,7 @@ public static boolean isAppInFinalState(RMApp rmApp) { || appState == RMAppState.KILLED; } - private RMAppState getRecoveredFinalState() { + public RMAppState getRecoveredFinalState() { return this.recoveredFinalState; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 48144c35c0..88a89b5854 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -83,6 +83,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppFailedAttemptEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptRegistrationEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptStatusupdateEvent; @@ -1102,6 +1103,9 @@ private static class AttemptRecoveredTransition @Override public RMAppAttemptState transition(RMAppAttemptImpl appAttempt, RMAppAttemptEvent event) { + RMApp rmApp = appAttempt.rmContext.getRMApps().get( + appAttempt.getAppAttemptId().getApplicationId()); + /* * If last attempt recovered final state is null .. it means attempt was * started but AM container may or may not have started / finished. @@ -1109,8 +1113,6 @@ public RMAppAttemptState transition(RMAppAttemptImpl appAttempt, */ if (appAttempt.recoveredFinalState != null) { appAttempt.progress = 1.0f; - RMApp rmApp =appAttempt.rmContext.getRMApps().get( - appAttempt.getAppAttemptId().getApplicationId()); if (appAttempt.submissionContext .getKeepContainersAcrossApplicationAttempts() @@ -1130,7 +1132,24 @@ public RMAppAttemptState transition(RMAppAttemptImpl appAttempt, appAttempt, event); } return appAttempt.recoveredFinalState; - } else { + } else if (RMAppImpl.isAppInFinalState(rmApp)) { + // Somehow attempt final state was not saved but app final state was saved. + // Skip adding the attempt into scheduler + RMAppState appState = ((RMAppImpl) rmApp).getRecoveredFinalState(); + LOG.warn(rmApp.getApplicationId() + " final state (" + appState + + ") was recorded, but " + appAttempt.applicationAttemptId + + " final state (" + appAttempt.recoveredFinalState + + ") was not recorded."); + switch (appState) { + case FINISHED: + return RMAppAttemptState.FINISHED; + case FAILED: + return RMAppAttemptState.FAILED; + case KILLED: + return RMAppAttemptState.KILLED; + } + return RMAppAttemptState.FAILED; + } else{ // Add the current attempt to the scheduler. if (appAttempt.rmContext.isWorkPreservingRecoveryEnabled()) { // Need to register an app attempt before AM can register @@ -1164,6 +1183,7 @@ public RMAppAttemptState transition(RMAppAttemptImpl appAttempt, } } + private void rememberTargetTransitions(RMAppAttemptEvent event, Object transitionToDo, RMAppAttemptState targetFinalState) { transitionTodo = transitionToDo; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java index 2ef427ed52..05713877c2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java @@ -59,6 +59,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart.TestSecurityMockRM; import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystemTestUtil; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -1367,4 +1368,44 @@ public void testContainerCompleteMsgNotLostAfterAMFailedAndRMRestart() throws Ex assertEquals(1,loadedAttempt1.getJustFinishedContainers().size()); } + // Test that if application state was saved, but attempt state was not saved. + // RM should start correctly. + @Test (timeout = 20000) + public void testAppStateSavedButAttemptStateNotSaved() throws Exception { + MemoryRMStateStore memStore = new MemoryRMStateStore() { + @Override public synchronized void updateApplicationAttemptStateInternal( + ApplicationAttemptId appAttemptId, + ApplicationAttemptStateData attemptState) { + // do nothing; + // simulate the failure that attempt final state is not saved. + } + }; + memStore.init(conf); + rm1 = new MockRM(conf, memStore); + rm1.start(); + + MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); + nm1.registerNode(); + + RMApp app1 = rm1.submitApp(200); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); + MockRM.finishAMAndVerifyAppState(app1, rm1, nm1, am1); + + ApplicationStateData appSavedState = + memStore.getState().getApplicationState().get(app1.getApplicationId()); + + // check that app state is saved. + assertEquals(RMAppState.FINISHED, appSavedState.getState()); + // check that attempt state is not saved. + assertNull(appSavedState.getAttempt(am1.getApplicationAttemptId()).getState()); + + rm2 = new MockRM(conf, memStore); + rm2.start(); + RMApp recoveredApp1 = + rm2.getRMContext().getRMApps().get(app1.getApplicationId()); + + assertEquals(RMAppState.FINISHED, recoveredApp1.getState()); + // check that attempt state is recovered correctly. + assertEquals(RMAppAttemptState.FINISHED, recoveredApp1.getCurrentAppAttempt().getState()); + } }