From b3056c266a628a65cf7ceb61b55ab3bd0a09baf2 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Wed, 15 Oct 2014 18:47:26 -0700 Subject: [PATCH] YARN-2685. Fixed a bug in CommonNodeLabelsManager that caused wrong resource tracking per label when a host runs multiple node-managers. Contributed by Wangda Tan. --- hadoop-yarn-project/CHANGES.txt | 4 +++ .../nodelabels/CommonNodeLabelsManager.java | 1 + .../nodelabels/TestRMNodeLabelsManager.java | 29 +++++++++++++++++++ 3 files changed, 34 insertions(+) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 97fea49052..b0e307ce7f 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -583,6 +583,10 @@ Release 2.6.0 - UNRELEASED YARN-2628. Capacity scheduler with DominantResourceCalculator carries out reservation even though slots are free. (Varun Vasudev via jianhe) + YARN-2685. Fixed a bug in CommonNodeLabelsManager that caused wrong resource + tracking per label when a host runs multiple node-managers. (Wangda Tan via + vinodkv) + BREAKDOWN OF YARN-1051 SUBTASKS AND RELATED JIRAS YARN-1707. Introduce APIs to add/remove/resize queues in the diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java index 89fbf09bca..511b5ee56a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java @@ -129,6 +129,7 @@ public Node copy() { if (labels != null) { c.labels = Collections.newSetFromMap(new ConcurrentHashMap()); + c.labels.addAll(labels); } else { c.labels = null; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java index 26284e2acb..81eead9b8e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.nodelabels; +import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -25,6 +26,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.nodelabels.NodeLabelTestBase; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; @@ -364,4 +366,31 @@ public void testGetQueueResource() throws Exception { Assert.assertEquals(clusterResource, mgr.getQueueResource("Q5", q5Label, clusterResource)); } + + @Test(timeout=5000) + public void testGetLabelResourceWhenMultipleNMsExistingInSameHost() throws IOException { + // active two NM to n1, one large and one small + mgr.activateNode(NodeId.newInstance("n1", 1), SMALL_RESOURCE); + mgr.activateNode(NodeId.newInstance("n1", 2), SMALL_RESOURCE); + mgr.activateNode(NodeId.newInstance("n1", 3), SMALL_RESOURCE); + mgr.activateNode(NodeId.newInstance("n1", 4), SMALL_RESOURCE); + + // check resource of no label, it should be small * 4 + Assert.assertEquals( + mgr.getResourceByLabel(CommonNodeLabelsManager.NO_LABEL, null), + Resources.multiply(SMALL_RESOURCE, 4)); + + // change two of these nodes to p1, check resource of no_label and P1 + mgr.addToCluserNodeLabels(toSet("p1")); + mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p1"), + toNodeId("n1:2"), toSet("p1"))); + + // check resource + Assert.assertEquals( + mgr.getResourceByLabel(CommonNodeLabelsManager.NO_LABEL, null), + Resources.multiply(SMALL_RESOURCE, 2)); + Assert.assertEquals( + mgr.getResourceByLabel("p1", null), + Resources.multiply(SMALL_RESOURCE, 2)); + } }