From 845c4e52bdd579a24df5dbba7477b0ebf2fa16f1 Mon Sep 17 00:00:00 2001 From: Rohith Sharma K S Date: Wed, 19 Jul 2017 11:10:52 +0530 Subject: [PATCH] YARN-6819. Application report fails if app rejected due to nodesize. Contributed by Bibin A Chundatt. --- .../recovery/RMStateStore.java | 5 ++-- .../resourcemanager/rmapp/RMAppEvent.java | 24 ------------------- .../resourcemanager/rmapp/RMAppEventType.java | 1 + .../resourcemanager/rmapp/RMAppImpl.java | 8 +++---- .../recovery/TestZKRMStateStore.java | 14 ++++++----- .../rmapp/TestRMAppTransitions.java | 17 +++++++++++++ 6 files changed, 33 insertions(+), 36 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java index e945b597cf..d0a8cf5ec4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java @@ -221,8 +221,9 @@ public RMStateStoreState transition(RMStateStore store, } catch (Exception e) { LOG.error("Error storing app: " + appId, e); if (e instanceof StoreLimitException) { - store.notifyApplication(new RMAppEvent(appId, - RMAppEventType.APP_REJECTED, e.getMessage(), false)); + store.notifyApplication( + new RMAppEvent(appId, RMAppEventType.APP_SAVE_FAILED, + e.getMessage())); } else { isFenced = store.notifyStoreOperationFailedInternal(e); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEvent.java index 0c6139e94e..5c46945754 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEvent.java @@ -25,7 +25,6 @@ public class RMAppEvent extends AbstractEvent{ private final ApplicationId appId; private final String diagnosticMsg; - private boolean storeAppInfo; public RMAppEvent(ApplicationId appId, RMAppEventType type) { this(appId, type, ""); @@ -36,21 +35,6 @@ public RMAppEvent(ApplicationId appId, RMAppEventType type, super(type); this.appId = appId; this.diagnosticMsg = diagnostic; - this.storeAppInfo = true; - } - - /** - * Constructor to create RM Application Event type. - * - * @param appId application Id - * @param type RM Event type - * @param diagnostic Diagnostic message for event - * @param storeApp Application should be saved or not - */ - public RMAppEvent(ApplicationId appId, RMAppEventType type, String diagnostic, - boolean storeApp) { - this(appId, type, diagnostic); - this.storeAppInfo = storeApp; } public ApplicationId getApplicationId() { @@ -61,12 +45,4 @@ public String getDiagnosticMsg() { return this.diagnosticMsg; } - /** - * Store application to state store or not. - * - * @return boolean application should be saved to store. - */ - public boolean doStoreAppInfo() { - return storeAppInfo; - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java index aa5d6f0971..04d2db522f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java @@ -47,4 +47,5 @@ public enum RMAppEventType { // Source: RMStateStore APP_NEW_SAVED, APP_UPDATE_SAVED, + APP_SAVE_FAILED, } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index bf8fa4f3a1..fa2f20c723 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -243,6 +243,8 @@ RMAppEventType.APP_NEW_SAVED, new AddApplicationToSchedulerTransition()) RMAppEventType.APP_REJECTED, new FinalSavingTransition(new AppRejectedTransition(), RMAppState.FAILED)) + .addTransition(RMAppState.NEW_SAVING, RMAppState.FAILED, + RMAppEventType.APP_SAVE_FAILED, new AppRejectedTransition()) // Transitions from SUBMITTED state .addTransition(RMAppState.SUBMITTED, RMAppState.SUBMITTED, @@ -1307,10 +1309,8 @@ public FinalSavingTransition(Object transitionToDo, @Override public void transition(RMAppImpl app, RMAppEvent event) { - if (event.doStoreAppInfo()) { - app.rememberTargetTransitionsAndStoreState(event, transitionToDo, - targetedFinalState, stateToBeStored); - } + app.rememberTargetTransitionsAndStoreState(event, transitionToDo, + targetedFinalState, stateToBeStored); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java index fcd8647638..942e9e88bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java @@ -270,20 +270,22 @@ public void testZKNodeLimit() throws Exception { GenericTestUtils.waitFor(new Supplier() { @Override public Boolean get() { - return dispatcher.apprejectedEvnt; + return dispatcher.appsavefailedEvnt; } }, 100, 5000); } static class TestAppRejDispatcher extends TestDispatcher { - private boolean apprejectedEvnt; + private boolean appsavefailedEvnt; public void handle(Event event) { - if (event instanceof RMAppEvent - && event.getType().equals(RMAppEventType.APP_REJECTED)) { - apprejectedEvnt = true; + if (event instanceof RMAppEvent && event.getType() + .equals(RMAppEventType.APP_SAVE_FAILED)) { + appsavefailedEvnt = true; } - }; + } + + ; } @Test (timeout = 60000) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index 5aa7af9e51..7c54b60b9b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -652,6 +652,23 @@ public void testAppNewSavingReject() throws IOException { verifyApplicationFinished(RMAppState.FAILED); } + @Test (timeout = 30000) + public void testAppNewSavingSaveReject() throws IOException { + LOG.info("--- START: testAppNewSavingSaveReject ---"); + RMApp application = testCreateAppNewSaving(null); + // NEW_SAVING => FAILED event RMAppEventType.APP_SAVE_FAILED + String rejectedText = "Test Application Rejected"; + RMAppEvent event = new RMAppEvent(application.getApplicationId(), + RMAppEventType.APP_SAVE_FAILED, rejectedText); + application.handle(event); + rmDispatcher.await(); + assertFailed(application, rejectedText); + verify(store, times(0)).updateApplicationState( + any(ApplicationStateData.class)); + verifyApplicationFinished(RMAppState.FAILED); + assertTimesAtFinish(application); + } + @Test (timeout = 30000) public void testAppSubmittedRejected() throws IOException { LOG.info("--- START: testAppSubmittedRejected ---");