big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-11395. RM UI, RMAttemptBlock can not render FINAL_SAVING. Contributed by Bence Kosztolnik

Browse Source 
- In the YARN-1345 remove of FINAL_SAVING was missed from RMAttemptBlock
- Same issue was present after YARN-1345 in YARN-4411
- YARN-4411 logic was applied in this commit for FINAL_SAVING
This commit is contained in:
Bence Kosztolnik 2022-12-14 15:58:02 +01:00 committed by Benjamin Teke
parent df093ef9af
commit bf8ab83cd0
7 changed files with 84 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/StateMachine.java
View File

@ -27,6 +27,7 @@ public interface StateMachine
<STATE extends Enum<STATE>,
EVENTTYPE extends Enum<EVENTTYPE>, EVENT> {
public STATE getCurrentState();
public STATE getPreviousState();
public STATE doTransition(EVENTTYPE eventType, EVENT event)
throws InvalidStateTransitionException;
}

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/state/StateMachineFactory.java
View File

@ -457,6 +457,7 @@ private class InternalStateMachine
implements StateMachine<STATE, EVENTTYPE, EVENT> {
private final OPERAND operand;
private STATE currentState;
private STATE previousState;
private final StateTransitionListener<OPERAND, EVENT, STATE> listener;
InternalStateMachine(OPERAND operand, STATE initialState) {
@ -479,14 +480,19 @@ public synchronized STATE getCurrentState() {
return currentState;
}
@Override
public synchronized STATE getPreviousState() {
return previousState;
}
@Override
public synchronized STATE doTransition(EVENTTYPE eventType, EVENT event)
throws InvalidStateTransitionException {
listener.preTransition(operand, currentState, event);
STATE oldState = currentState;
previousState = currentState;
currentState = StateMachineFactory.this.doTransition
(operand, currentState, eventType, event);
listener.postTransition(operand, oldState, currentState, event);
listener.postTransition(operand, previousState, currentState, event);
return currentState;
}
}

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
View File

@ -453,6 +453,17 @@ public static YarnApplicationState createApplicationState(
}
}
public static YarnApplicationAttemptState convertRmAppAttemptStateToYarnApplicationAttemptState(
RMAppAttemptState currentState,
RMAppAttemptState previousState
) {
return createApplicationAttemptState(
currentState == RMAppAttemptState.FINAL_SAVING
? previousState
: currentState
);
}
public static YarnApplicationAttemptState createApplicationAttemptState(
RMAppAttemptState rmAppAttemptState) {
switch (rmAppAttemptState) {

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
View File

@ -206,6 +206,14 @@ public interface RMAppAttempt extends EventHandler<RMAppAttemptEvent> {
*/
RMAppAttemptState getState();
/**
* The previous state of the {@link RMAppAttempt} before the current state.
*
* @return the previous state of the {@link RMAppAttempt} before the current state
* for this application attempt.
*/
RMAppAttemptState getPreviousState();
/**
* Create the external user-facing state of the attempt of ApplicationMaster
* from the current state of the {@link RMAppAttempt}.

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
View File

@ -2220,13 +2220,22 @@ public RMAppAttemptState getState() {
}
@Override
public YarnApplicationAttemptState createApplicationAttemptState() {
RMAppAttemptState state = getState();
// If AppAttempt is in FINAL_SAVING state, return its previous state.
if (state.equals(RMAppAttemptState.FINAL_SAVING)) {
state = stateBeforeFinalSaving;
public RMAppAttemptState getPreviousState() {
this.readLock.lock();
try {
return this.stateMachine.getPreviousState();
} finally {
this.readLock.unlock();
}
return RMServerUtils.createApplicationAttemptState(state);
}
@Override
public YarnApplicationAttemptState createApplicationAttemptState() {
return RMServerUtils.convertRmAppAttemptStateToYarnApplicationAttemptState(
getState(),
stateBeforeFinalSaving
);
}
private void launchAttempt(){

25
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
View File

@ -39,6 +39,7 @@
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
@ -437,20 +438,24 @@ private boolean isApplicationInFinalState(YarnApplicationAttemptState state) {
@Override
protected void createAttemptHeadRoomTable(Block html) {
RMAppAttempt attempt = getRMAppAttempt();
if (attempt != null) {
if (!isApplicationInFinalState(YarnApplicationAttemptState
.valueOf(attempt.getAppAttemptState().toString()))) {
RMAppAttemptMetrics metrics = attempt.getRMAppAttemptMetrics();
DIV<Hamlet> pdiv = html.__(InfoBlock.class).div(_INFO_WRAP);
info("Application Attempt Overview").clear();
info("Application Attempt Metrics").__(
if (attempt != null && !isApplicationInFinalState(createApplicationAttemptState(attempt))) {
RMAppAttemptMetrics metrics = attempt.getRMAppAttemptMetrics();
DIV<Hamlet> pdiv = html.__(InfoBlock.class).div(_INFO_WRAP);
info("Application Attempt Overview").clear();
info("Application Attempt Metrics").__(
"Application Attempt Headroom : ", metrics == null ? "N/A" :
metrics.getApplicationAttemptHeadroom());
pdiv.__();
}
metrics.getApplicationAttemptHeadroom());
pdiv.__();
}
}
private YarnApplicationAttemptState createApplicationAttemptState(RMAppAttempt attempt) {
return RMServerUtils.convertRmAppAttemptStateToYarnApplicationAttemptState(
attempt.getState(),
attempt.getPreviousState()
);
}
private RMAppAttempt getRMAppAttempt() {
ApplicationId appId = this.appAttemptId.getApplicationId();
RMAppAttempt attempt = null;

26
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMServerUtils.java
View File

@ -43,10 +43,12 @@
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.UpdateContainerError;
import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.impl.pb.UpdateContainerRequestPBImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
@ -409,6 +411,30 @@ public void testGetApplicableNodeCountForAMLocalityAndLabels()
Assert.assertEquals(15,
RMServerUtils.getApplicableNodeCountForAM(rmContext, conf, reqs));
}
@Test
public void testConvertRmAppAttemptStateToYarnApplicationAttemptState() {
Assert.assertEquals(
YarnApplicationAttemptState.FAILED,
RMServerUtils.convertRmAppAttemptStateToYarnApplicationAttemptState(
RMAppAttemptState.FINAL_SAVING,
RMAppAttemptState.FAILED
)
);
Assert.assertEquals(
YarnApplicationAttemptState.SCHEDULED,
RMServerUtils.convertRmAppAttemptStateToYarnApplicationAttemptState(
RMAppAttemptState.FINAL_SAVING,
RMAppAttemptState.SCHEDULED
)
);
Assert.assertEquals(
YarnApplicationAttemptState.NEW,
RMServerUtils.convertRmAppAttemptStateToYarnApplicationAttemptState(
RMAppAttemptState.NEW,
null
)
);
}
private ResourceRequest createResourceRequest(String resource,
boolean relaxLocality, String nodeLabel) {