big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10649. Fix RMNodeImpl.updateExistContainers leak (#2719). Contributed by Max Xie

Browse Source 
(cherry picked from commit d615e2d3bd)
This commit is contained in:
Neil 2021-03-04 17:22:58 +08:00 committed by bibinchundatt
parent 4462da0a84
commit 0396a721e3
2 changed files with 17 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
View File

@ -1464,6 +1464,11 @@ public int getQueueSize() {
return nodeUpdateQueue.size(); return nodeUpdateQueue.size();
} }
// For test only.
@VisibleForTesting
public Map<ContainerId, ContainerStatus> getUpdatedExistContainers() {
return this.updatedExistContainers;
}
// For test only. // For test only.
@VisibleForTesting @VisibleForTesting
public Set<ContainerId> getLaunchedContainers() { public Set<ContainerId> getLaunchedContainers() {
@ -1582,6 +1587,7 @@ private void handleContainerStatus(List<ContainerStatus> containerStatuses) {
} else { } else {
// A finished container // A finished container
launchedContainers.remove(containerId); launchedContainers.remove(containerId);
updatedExistContainers.remove(containerId);
if (completedContainers.add(containerId)) { if (completedContainers.add(containerId)) {
newlyCompletedContainers.add(remoteContainer); newlyCompletedContainers.add(remoteContainer);
} }
@ -1595,6 +1601,7 @@ private void handleContainerStatus(List<ContainerStatus> containerStatuses) {
findLostContainers(numRemoteRunningContainers, containerStatuses); findLostContainers(numRemoteRunningContainers, containerStatuses);
for (ContainerStatus remoteContainer : lostContainers) { for (ContainerStatus remoteContainer : lostContainers) {
ContainerId containerId = remoteContainer.getContainerId(); ContainerId containerId = remoteContainer.getContainerId();
updatedExistContainers.remove(containerId);
if (completedContainers.add(containerId)) { if (completedContainers.add(containerId)) {
newlyCompletedContainers.add(remoteContainer); newlyCompletedContainers.add(remoteContainer);
} }

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
View File

@ -1096,6 +1096,12 @@ public void testDisappearingContainer() {
node.getLaunchedContainers().contains(cid1)); node.getLaunchedContainers().contains(cid1));
Assert.assertTrue("second container not running", Assert.assertTrue("second container not running",
node.getLaunchedContainers().contains(cid2)); node.getLaunchedContainers().contains(cid2));
assertEquals("unexpected number of running containers",
2, node.getUpdatedExistContainers().size());
Assert.assertTrue("first container not running",
node.getUpdatedExistContainers().containsKey(cid1));
Assert.assertTrue("second container not running",
node.getUpdatedExistContainers().containsKey(cid2));
assertEquals("already completed containers", assertEquals("already completed containers",
0, completedContainers.size()); 0, completedContainers.size());
containerStats.remove(0); containerStats.remove(0);
@ -1115,6 +1121,10 @@ public void testDisappearingContainer() {
1, node.getLaunchedContainers().size()); 1, node.getLaunchedContainers().size());
Assert.assertTrue("second container not running", Assert.assertTrue("second container not running",
node.getLaunchedContainers().contains(cid2)); node.getLaunchedContainers().contains(cid2));
assertEquals("unexpected number of running containers",
1, node.getUpdatedExistContainers().size());
Assert.assertTrue("second container not running",
node.getUpdatedExistContainers().containsKey(cid2));
} }
@Test @Test