big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-17624. Fix DFSNetworkTopology#chooseRandomWithStorageType() availableCount when excluded node is not in selected scope. (#7042). Contributed by fuchaohong.

Browse Source 
Signed-off-by: He Xiaoqiao <hexiaoqiao@apache.org>
This commit is contained in:
fuchaohong 2024-09-27 16:56:09 +08:00 committed by GitHub
parent 21cdb450ef
commit 3d81dde28b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 21 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java
View File

@ -219,7 +219,8 @@ Node chooseRandomWithStorageType(final String scope,
}
if (excludedNodes != null) {
for (Node excludedNode : excludedNodes) {
if (excludeRoot != null && isNodeInScope(excludedNode, excludedScope)) {
if ((excludeRoot != null && isNodeInScope(excludedNode, excludedScope)) ||
!isNodeInScope(excludedNode, scope)) {
continue;
}
if (excludedNode instanceof DatanodeDescriptor) {

19
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/net/TestDFSNetworkTopology.java
View File

@ -667,4 +667,23 @@ public void testChooseRandomWithStorageTypeScopeEqualsExcludedNodes() {
null, excluded, StorageType.DISK);
assertNull("No node should have been selected.", n);
}
@Test
public void testChooseRandomWithStorageTypeWithExcludeNodes() {
DFSNetworkTopology dfsCluster =
DFSNetworkTopology.getInstance(new Configuration());
final String[] racks = {"/default/rack1", "/default/rack2"};
final String[] hosts = {"host1", "host2"};
final StorageType[] types = {StorageType.DISK, StorageType.DISK};
final DatanodeStorageInfo[] storages =
DFSTestUtil.createDatanodeStorageInfos(2, racks, hosts, types);
DatanodeDescriptor[] dns = DFSTestUtil.toDatanodeDescriptor(storages);
dfsCluster.add(dns[0]);
dfsCluster.add(dns[1]);
HashSet<Node> excluded = new HashSet<>();
excluded.add(dns[1]);
Node n = dfsCluster.chooseRandomWithStorageType("/default/rack1",
null, excluded, StorageType.DISK);
assertNotNull("/default/rack1/host1 should be selected.", n);
}
}