big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-13666. Supporting rack exclusion in countNumOfAvailableNodes in NetworkTopology. Contributed by Inigo Goiri.

Browse Source
This commit is contained in:
Ayush Saxena 2020-02-18 00:29:21 +05:30
parent 439d935e1d
commit 84f7638840
2 changed files with 23 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
View File

@ -670,7 +670,11 @@ public int countNumOfAvailableNodes(String scope,
} }
if ((NodeBase.getPath(node) + NodeBase.PATH_SEPARATOR_STR) if ((NodeBase.getPath(node) + NodeBase.PATH_SEPARATOR_STR)
.startsWith(scope + NodeBase.PATH_SEPARATOR_STR)) { .startsWith(scope + NodeBase.PATH_SEPARATOR_STR)) {
excludedCountInScope++; if (node instanceof InnerNode) {
excludedCountInScope += ((InnerNode) node).getNumOfLeaves();
} else {
excludedCountInScope++;
}
} else { } else {
excludedCountOffScope++; excludedCountOffScope++;
} }

18
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java
View File

@ -614,4 +614,22 @@ public void testChooseRandomInclude3() {
frequency.get(dataNodes[i]) > 0); frequency.get(dataNodes[i]) > 0);
} }
} }
@Test
public void testCountNumOfAvailableNodes() {
int numNodes = cluster.countNumOfAvailableNodes(NodeBase.ROOT, null);
assertEquals(20, numNodes);
// Excluding a single node
Collection<Node> excludedNodes = new HashSet<Node>();
excludedNodes.add(dataNodes[0]);
numNodes = cluster.countNumOfAvailableNodes(NodeBase.ROOT, excludedNodes);
assertEquals(19, numNodes);
// Excluding a full rack
Node d4r1 = cluster.getNode("/d4/r1");
excludedNodes.add(d4r1);
numNodes = cluster.countNumOfAvailableNodes(NodeBase.ROOT, excludedNodes);
assertEquals(12, numNodes);
}
} }