big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-16028. Fix NetworkTopology chooseRandom function to support excluded nodes. Contributed by Sihai Ke.

Browse Source
This commit is contained in:
Inigo Goiri 2019-01-04 09:55:09 -08:00
parent ddc0a40507
commit f4e18242bd
2 changed files with 42 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
View File

@ -532,8 +532,13 @@ protected Node chooseRandom(final String scope, String excludedScope,
if (excludedScope == null) { if (excludedScope == null) {
availableNodes = countNumOfAvailableNodes(scope, excludedNodes); availableNodes = countNumOfAvailableNodes(scope, excludedNodes);
} else { } else {
availableNodes = netlock.readLock().lock();
countNumOfAvailableNodes("~" + excludedScope, excludedNodes); try {
availableNodes = countNumOfAvailableNodes(scope, excludedNodes) -
countNumOfAvailableNodes(excludedScope, excludedNodes);
} finally {
netlock.readLock().unlock();
}
} }
LOG.debug("Choosing random from {} available nodes on node {}," LOG.debug("Choosing random from {} available nodes on node {},"
+ " scope={}, excludedScope={}, excludeNodes={}. numOfDatanodes={}.", + " scope={}, excludedScope={}, excludeNodes={}. numOfDatanodes={}.",

35
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java
View File

@ -20,6 +20,7 @@
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Arrays;
import org.apache.commons.math3.stat.inference.ChiSquareTest; import org.apache.commons.math3.stat.inference.ChiSquareTest;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
@ -194,6 +195,40 @@ public void testChooseRandom() {
2, histogram.size()); 2, histogram.size());
} }
@Test
public void testChooseRandomExcluded() {
// create the topology
// a1
// b1------|--------b2
// | |
// c1-----|-----c2 c3
// / \ | |
// / \ | |
// node1 node2 node3 node4
NetworkTopology cluster = NetworkTopology.getInstance(new Configuration());
NodeElement node1 = getNewNode("node1", "/a1/b1/c1");
cluster.add(node1);
NodeElement node2 = getNewNode("node2", "/a1/b1/c1");
cluster.add(node2);
NodeElement node3 = getNewNode("node3", "/a1/b1/c2");
cluster.add(node3);
NodeElement node4 = getNewNode("node4", "/a1/b2/c3");
cluster.add(node4);
Node node = cluster.chooseRandom("/a1/b1", "/a1/b1/c1", null);
assertSame("node3", node.getName());
node = cluster.chooseRandom("/a1/b1", "/a1/b1/c1", Arrays.asList(node1));
assertSame("node3", node.getName());
node = cluster.chooseRandom("/a1/b1", "/a1/b1/c1", Arrays.asList(node3));
assertNull(node);
node = cluster.chooseRandom("/a1/b1", "/a1/b1/c1", Arrays.asList(node4));
assertSame("node3", node.getName());
}
private NodeElement getNewNode(String name, String rackLocation) { private NodeElement getNewNode(String name, String rackLocation) {
NodeElement node = new NodeElement(name); NodeElement node = new NodeElement(name);
node.setNetworkLocation(rackLocation); node.setNetworkLocation(rackLocation);