From 3d81dde28b637f9015a2f4efeeb4b21a212fff74 Mon Sep 17 00:00:00 2001 From: fuchaohong <1783129294@qq.com> Date: Fri, 27 Sep 2024 16:56:09 +0800 Subject: [PATCH] HDFS-17624. Fix DFSNetworkTopology#chooseRandomWithStorageType() availableCount when excluded node is not in selected scope. (#7042). Contributed by fuchaohong. Signed-off-by: He Xiaoqiao --- .../hadoop/hdfs/net/DFSNetworkTopology.java | 3 ++- .../hdfs/net/TestDFSNetworkTopology.java | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java index 77e610434b..66288a749b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java @@ -219,7 +219,8 @@ Node chooseRandomWithStorageType(final String scope, } if (excludedNodes != null) { for (Node excludedNode : excludedNodes) { - if (excludeRoot != null && isNodeInScope(excludedNode, excludedScope)) { + if ((excludeRoot != null && isNodeInScope(excludedNode, excludedScope)) || + !isNodeInScope(excludedNode, scope)) { continue; } if (excludedNode instanceof DatanodeDescriptor) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/net/TestDFSNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/net/TestDFSNetworkTopology.java index 6681b2e627..fb41651d68 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/net/TestDFSNetworkTopology.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/net/TestDFSNetworkTopology.java @@ -667,4 +667,23 @@ public void testChooseRandomWithStorageTypeScopeEqualsExcludedNodes() { null, excluded, StorageType.DISK); assertNull("No node should have been selected.", n); } + + @Test + public void testChooseRandomWithStorageTypeWithExcludeNodes() { + DFSNetworkTopology dfsCluster = + DFSNetworkTopology.getInstance(new Configuration()); + final String[] racks = {"/default/rack1", "/default/rack2"}; + final String[] hosts = {"host1", "host2"}; + final StorageType[] types = {StorageType.DISK, StorageType.DISK}; + final DatanodeStorageInfo[] storages = + DFSTestUtil.createDatanodeStorageInfos(2, racks, hosts, types); + DatanodeDescriptor[] dns = DFSTestUtil.toDatanodeDescriptor(storages); + dfsCluster.add(dns[0]); + dfsCluster.add(dns[1]); + HashSet excluded = new HashSet<>(); + excluded.add(dns[1]); + Node n = dfsCluster.chooseRandomWithStorageType("/default/rack1", + null, excluded, StorageType.DISK); + assertNotNull("/default/rack1/host1 should be selected.", n); + } }