From bf8ab83cd00fcbbc77e96b9ae41f4664ec646965 Mon Sep 17 00:00:00 2001 From: Bence Kosztolnik Date: Wed, 14 Dec 2022 15:58:02 +0100 Subject: [PATCH] YARN-11395. RM UI, RMAttemptBlock can not render FINAL_SAVING. Contributed by Bence Kosztolnik - In the YARN-1345 remove of FINAL_SAVING was missed from RMAttemptBlock - Same issue was present after YARN-1345 in YARN-4411 - YARN-4411 logic was applied in this commit for FINAL_SAVING --- .../hadoop/yarn/state/StateMachine.java | 1 + .../yarn/state/StateMachineFactory.java | 10 +++++-- .../server/resourcemanager/RMServerUtils.java | 11 ++++++++ .../rmapp/attempt/RMAppAttempt.java | 8 ++++++ .../rmapp/attempt/RMAppAttemptImpl.java | 21 ++++++++++----- .../webapp/RMAppAttemptBlock.java | 25 +++++++++++------- .../resourcemanager/TestRMServerUtils.java | 26 +++++++++++++++++++ 7 files changed, 84 insertions(+), 18 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/StateMachine.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/StateMachine.java index 5151559630..083869ef3d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/StateMachine.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/StateMachine.java @@ -27,6 +27,7 @@ public interface StateMachine , EVENTTYPE extends Enum, EVENT> { public STATE getCurrentState(); + public STATE getPreviousState(); public STATE doTransition(EVENTTYPE eventType, EVENT event) throws InvalidStateTransitionException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/StateMachineFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/StateMachineFactory.java index e7660c14b7..3ac04a2b5b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/StateMachineFactory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/StateMachineFactory.java @@ -457,6 +457,7 @@ private class InternalStateMachine implements StateMachine { private final OPERAND operand; private STATE currentState; + private STATE previousState; private final StateTransitionListener listener; InternalStateMachine(OPERAND operand, STATE initialState) { @@ -479,14 +480,19 @@ public synchronized STATE getCurrentState() { return currentState; } + @Override + public synchronized STATE getPreviousState() { + return previousState; + } + @Override public synchronized STATE doTransition(EVENTTYPE eventType, EVENT event) throws InvalidStateTransitionException { listener.preTransition(operand, currentState, event); - STATE oldState = currentState; + previousState = currentState; currentState = StateMachineFactory.this.doTransition (operand, currentState, eventType, event); - listener.postTransition(operand, oldState, currentState, event); + listener.postTransition(operand, previousState, currentState, event); return currentState; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java index 79115902bc..de7a762660 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java @@ -453,6 +453,17 @@ public static YarnApplicationState createApplicationState( } } + public static YarnApplicationAttemptState convertRmAppAttemptStateToYarnApplicationAttemptState( + RMAppAttemptState currentState, + RMAppAttemptState previousState + ) { + return createApplicationAttemptState( + currentState == RMAppAttemptState.FINAL_SAVING + ? previousState + : currentState + ); + } + public static YarnApplicationAttemptState createApplicationAttemptState( RMAppAttemptState rmAppAttemptState) { switch (rmAppAttemptState) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java index cfd91e902c..5d78c8b354 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java @@ -206,6 +206,14 @@ public interface RMAppAttempt extends EventHandler { */ RMAppAttemptState getState(); + /** + * The previous state of the {@link RMAppAttempt} before the current state. + * + * @return the previous state of the {@link RMAppAttempt} before the current state + * for this application attempt. + */ + RMAppAttemptState getPreviousState(); + /** * Create the external user-facing state of the attempt of ApplicationMaster * from the current state of the {@link RMAppAttempt}. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 5c34fe8f91..6010bd21a1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -2220,13 +2220,22 @@ public RMAppAttemptState getState() { } @Override - public YarnApplicationAttemptState createApplicationAttemptState() { - RMAppAttemptState state = getState(); - // If AppAttempt is in FINAL_SAVING state, return its previous state. - if (state.equals(RMAppAttemptState.FINAL_SAVING)) { - state = stateBeforeFinalSaving; + public RMAppAttemptState getPreviousState() { + this.readLock.lock(); + + try { + return this.stateMachine.getPreviousState(); + } finally { + this.readLock.unlock(); } - return RMServerUtils.createApplicationAttemptState(state); + } + + @Override + public YarnApplicationAttemptState createApplicationAttemptState() { + return RMServerUtils.convertRmAppAttemptStateToYarnApplicationAttemptState( + getState(), + stateBeforeFinalSaving + ); } private void launchAttempt(){ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java index 06763d5070..34dc0b5c48 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java @@ -39,6 +39,7 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; @@ -437,20 +438,24 @@ private boolean isApplicationInFinalState(YarnApplicationAttemptState state) { @Override protected void createAttemptHeadRoomTable(Block html) { RMAppAttempt attempt = getRMAppAttempt(); - if (attempt != null) { - if (!isApplicationInFinalState(YarnApplicationAttemptState - .valueOf(attempt.getAppAttemptState().toString()))) { - RMAppAttemptMetrics metrics = attempt.getRMAppAttemptMetrics(); - DIV pdiv = html.__(InfoBlock.class).div(_INFO_WRAP); - info("Application Attempt Overview").clear(); - info("Application Attempt Metrics").__( + if (attempt != null && !isApplicationInFinalState(createApplicationAttemptState(attempt))) { + RMAppAttemptMetrics metrics = attempt.getRMAppAttemptMetrics(); + DIV pdiv = html.__(InfoBlock.class).div(_INFO_WRAP); + info("Application Attempt Overview").clear(); + info("Application Attempt Metrics").__( "Application Attempt Headroom : ", metrics == null ? "N/A" : - metrics.getApplicationAttemptHeadroom()); - pdiv.__(); - } + metrics.getApplicationAttemptHeadroom()); + pdiv.__(); } } + private YarnApplicationAttemptState createApplicationAttemptState(RMAppAttempt attempt) { + return RMServerUtils.convertRmAppAttemptStateToYarnApplicationAttemptState( + attempt.getState(), + attempt.getPreviousState() + ); + } + private RMAppAttempt getRMAppAttempt() { ApplicationId appId = this.appAttemptId.getApplicationId(); RMAppAttempt attempt = null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java index 78fa90ded2..95cc983392 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java @@ -43,10 +43,12 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.UpdateContainerError; import org.apache.hadoop.yarn.api.records.UpdateContainerRequest; +import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.api.records.impl.pb.UpdateContainerRequestPBImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates; @@ -409,6 +411,30 @@ public void testGetApplicableNodeCountForAMLocalityAndLabels() Assert.assertEquals(15, RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs)); } + @Test + public void testConvertRmAppAttemptStateToYarnApplicationAttemptState() { + Assert.assertEquals( + YarnApplicationAttemptState.FAILED, + RMServerUtils.convertRmAppAttemptStateToYarnApplicationAttemptState( + RMAppAttemptState.FINAL_SAVING, + RMAppAttemptState.FAILED + ) + ); + Assert.assertEquals( + YarnApplicationAttemptState.SCHEDULED, + RMServerUtils.convertRmAppAttemptStateToYarnApplicationAttemptState( + RMAppAttemptState.FINAL_SAVING, + RMAppAttemptState.SCHEDULED + ) + ); + Assert.assertEquals( + YarnApplicationAttemptState.NEW, + RMServerUtils.convertRmAppAttemptStateToYarnApplicationAttemptState( + RMAppAttemptState.NEW, + null + ) + ); + } private ResourceRequest createResourceRequest(String resource, boolean relaxLocality, String nodeLabel) {