big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-2685. Fixed a bug in CommonNodeLabelsManager that caused wrong resource tracking per label when a host runs multiple node-managers. Contributed by Wangda Tan.

Browse Source
This commit is contained in:
Vinod Kumar Vavilapalli 2014-10-15 18:47:26 -07:00
parent f2ea555ac6
commit b3056c266a
3 changed files with 34 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-yarn-project/CHANGES.txt
View File

@ -583,6 +583,10 @@ Release 2.6.0 - UNRELEASED
YARN-2628. Capacity scheduler with DominantResourceCalculator carries out
reservation even though slots are free. (Varun Vasudev via jianhe)
YARN-2685. Fixed a bug in CommonNodeLabelsManager that caused wrong resource
tracking per label when a host runs multiple node-managers. (Wangda Tan via
vinodkv)
BREAKDOWN OF YARN-1051 SUBTASKS AND RELATED JIRAS
YARN-1707. Introduce APIs to add/remove/resize queues in the

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
View File

@ -129,6 +129,7 @@ public Node copy() {
if (labels != null) {
c.labels =
Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
c.labels.addAll(labels);
} else {
c.labels = null;
}

29
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.nodelabels;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
@ -25,6 +26,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.nodelabels.NodeLabelTestBase;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.After;
@ -364,4 +366,31 @@ public void testGetQueueResource() throws Exception {
Assert.assertEquals(clusterResource,
mgr.getQueueResource("Q5", q5Label, clusterResource));
}
@Test(timeout=5000)
public void testGetLabelResourceWhenMultipleNMsExistingInSameHost() throws IOException {
// active two NM to n1, one large and one small
mgr.activateNode(NodeId.newInstance("n1", 1), SMALL_RESOURCE);
mgr.activateNode(NodeId.newInstance("n1", 2), SMALL_RESOURCE);
mgr.activateNode(NodeId.newInstance("n1", 3), SMALL_RESOURCE);
mgr.activateNode(NodeId.newInstance("n1", 4), SMALL_RESOURCE);
// check resource of no label, it should be small * 4
Assert.assertEquals(
mgr.getResourceByLabel(CommonNodeLabelsManager.NO_LABEL, null),
Resources.multiply(SMALL_RESOURCE, 4));
// change two of these nodes to p1, check resource of no_label and P1
mgr.addToCluserNodeLabels(toSet("p1"));
mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p1"),
toNodeId("n1:2"), toSet("p1")));
// check resource
Assert.assertEquals(
mgr.getResourceByLabel(CommonNodeLabelsManager.NO_LABEL, null),
Resources.multiply(SMALL_RESOURCE, 2));
Assert.assertEquals(
mgr.getResourceByLabel("p1", null),
Resources.multiply(SMALL_RESOURCE, 2));
}
}