From 9f1655baf283075389dbe7506b4475881a47baf7 Mon Sep 17 00:00:00 2001 From: Peter Bacsko Date: Thu, 1 Apr 2021 11:39:08 +0200 Subject: [PATCH] YARN-9618. NodesListManager event improvement. Contributed by Qi Zhu. --- .../resourcemanager/NodesListManager.java | 8 +- .../rmapp/TestNodesListManager.java | 78 ++++++------------- 2 files changed, 26 insertions(+), 60 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java index f9e159168c..07d78cb1c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java @@ -509,12 +509,8 @@ private void sendRMAppNodeUpdateEventToNonFinalizedApps( RMNode eventNode, RMAppNodeUpdateType appNodeUpdateType) { for(RMApp app : rmContext.getRMApps().values()) { if (!app.isAppFinalStateStored()) { - this.rmContext - .getDispatcher() - .getEventHandler() - .handle( - new RMAppNodeUpdateEvent(app.getApplicationId(), eventNode, - appNodeUpdateType)); + app.handle(new RMAppNodeUpdateEvent(app.getApplicationId(), eventNode, + appNodeUpdateType)); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestNodesListManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestNodesListManager.java index 0df295c943..35148e9a45 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestNodesListManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestNodesListManager.java @@ -22,10 +22,7 @@ import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.spy; -import java.util.ArrayList; - import org.apache.hadoop.test.GenericTestUtils; -import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -52,8 +49,8 @@ import org.mockito.ArgumentMatcher; public class TestNodesListManager { - // To hold list of application for which event was received - ArrayList applist = new ArrayList(); + private boolean isRMAppEvent; + private boolean isNodesListEvent; @Test(timeout = 300000) public void testNodeUsableEvent() throws Exception { @@ -68,67 +65,32 @@ protected Dispatcher createDispatcher() { }; rm.start(); MockNM nm1 = rm.registerNode("h1:1234", 28000); - NodesListManager nodesListManager = rm.getNodesListManager(); Resource clusterResource = Resource.newInstance(28000, 8); RMNode rmnode = MockNodes.newNodeInfo(1, clusterResource); // Create killing APP - RMApp killrmApp = MockRMAppSubmitter.submitWithMemory(200, rm); - rm.killApp(killrmApp.getApplicationId()); - rm.waitForState(killrmApp.getApplicationId(), RMAppState.KILLED); + RMApp killRmApp = MockRMAppSubmitter.submitWithMemory(200, rm); + rm.killApp(killRmApp.getApplicationId()); + rm.waitForState(killRmApp.getApplicationId(), RMAppState.KILLED); // Create finish APP - RMApp finshrmApp = MockRMAppSubmitter.submitWithMemory(2000, rm); + RMApp finshRmApp = MockRMAppSubmitter.submitWithMemory(2000, rm); nm1.nodeHeartbeat(true); - RMAppAttempt attempt = finshrmApp.getCurrentAppAttempt(); + RMAppAttempt attempt = finshRmApp.getCurrentAppAttempt(); MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId()); am.registerAppAttempt(); am.unregisterAppAttempt(); nm1.nodeHeartbeat(attempt.getAppAttemptId(), 1, ContainerState.COMPLETE); rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FINISHED); - // Create submitted App - RMApp subrmApp = MockRMAppSubmitter.submitWithMemory(200, rm); - // Fire Event for NODE_USABLE - nodesListManager.handle(new NodesListManagerEvent( + // Should not have RMAppNodeUpdateEvent to AsyncDispatcher. + dispatcher.getEventHandler().handle(new NodesListManagerEvent( NodesListManagerEventType.NODE_USABLE, rmnode)); - if (applist.size() > 0) { - Assert.assertTrue( - "Event based on running app expected " + subrmApp.getApplicationId(), - applist.contains(subrmApp.getApplicationId())); - Assert.assertFalse( - "Event based on finish app not expected " - + finshrmApp.getApplicationId(), - applist.contains(finshrmApp.getApplicationId())); - Assert.assertFalse( - "Event based on killed app not expected " - + killrmApp.getApplicationId(), - applist.contains(killrmApp.getApplicationId())); - } else { - Assert.fail("Events received should have beeen more than 1"); - } - applist.clear(); - - // Fire Event for NODE_UNUSABLE - nodesListManager.handle(new NodesListManagerEvent( - NodesListManagerEventType.NODE_UNUSABLE, rmnode)); - if (applist.size() > 0) { - Assert.assertTrue( - "Event based on running app expected " + subrmApp.getApplicationId(), - applist.contains(subrmApp.getApplicationId())); - Assert.assertFalse( - "Event based on finish app not expected " - + finshrmApp.getApplicationId(), - applist.contains(finshrmApp.getApplicationId())); - Assert.assertFalse( - "Event based on killed app not expected " - + killrmApp.getApplicationId(), - applist.contains(killrmApp.getApplicationId())); - } else { - Assert.fail("Events received should have beeen more than 1"); - } - + Assert.assertFalse("Got unexpected RM app event", + getIsRMAppEvent()); + Assert.assertTrue("Received no NodesListManagerEvent", + getIsNodesListEvent()); } @Test @@ -241,9 +203,10 @@ class EventArgMatcher implements ArgumentMatcher { @Override public boolean matches(AbstractEvent argument) { if (argument instanceof RMAppNodeUpdateEvent) { - ApplicationId appid = - ((RMAppNodeUpdateEvent) argument).getApplicationId(); - applist.add(appid); + isRMAppEvent = true; + } + if (argument instanceof NodesListManagerEvent) { + isNodesListEvent = true; } return false; } @@ -256,4 +219,11 @@ public boolean matches(AbstractEvent argument) { }; } + public boolean getIsNodesListEvent() { + return isNodesListEvent; + } + + public boolean getIsRMAppEvent() { + return isRMAppEvent; + } }