diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index b4b9d97cf4..b4dab4d3bb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -21,6 +21,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; +import org.apache.hadoop.hdfs.net.DFSNetworkTopology; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault; import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyRackFaultTolerant; @@ -1177,6 +1178,12 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.use.dfs.network.topology"; public static final boolean DFS_USE_DFS_NETWORK_TOPOLOGY_DEFAULT = true; + public static final String DFS_NET_TOPOLOGY_IMPL_KEY = + "dfs.net.topology.impl"; + + public static final Class DFS_NET_TOPOLOGY_IMPL_DEFAULT = + DFSNetworkTopology.class; + // dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry @Deprecated public static final String DFS_CLIENT_RETRY_POLICY_ENABLED_KEY diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java index e74cdecbd4..7889ef4967 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java @@ -22,11 +22,13 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.net.Node; import org.apache.hadoop.net.NodeBase; +import org.apache.hadoop.util.ReflectionUtils; import java.util.ArrayList; import java.util.Collection; @@ -44,8 +46,12 @@ public class DFSNetworkTopology extends NetworkTopology { private static final Random RANDOM = new Random(); public static DFSNetworkTopology getInstance(Configuration conf) { - DFSNetworkTopology nt = new DFSNetworkTopology(); - return (DFSNetworkTopology)nt.init(DFSTopologyNodeImpl.FACTORY); + + DFSNetworkTopology nt = ReflectionUtils.newInstance(conf.getClass( + DFSConfigKeys.DFS_NET_TOPOLOGY_IMPL_KEY, + DFSConfigKeys.DFS_NET_TOPOLOGY_IMPL_DEFAULT, + DFSNetworkTopology.class), conf); + return (DFSNetworkTopology) nt.init(DFSTopologyNodeImpl.FACTORY); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index f8cce60c7e..4ed4690836 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -4859,6 +4859,20 @@ true Enables DFSNetworkTopology to choose nodes for placing replicas. + When enabled, NetworkTopology will be instantiated as class defined in + property dfs.net.topology.impl, otherwise NetworkTopology will be + instantiated as class defined in property net.topology.impl. + + + + + dfs.net.topology.impl + org.apache.hadoop.hdfs.net.DFSNetworkTopology + + The implementation class of NetworkTopology used in HDFS. By default, + the class org.apache.hadoop.hdfs.net.DFSNetworkTopology is specified and + used in block placement. + This property only works when dfs.use.dfs.network.topology is true. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java index 96841cf515..f70874015e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java @@ -41,6 +41,8 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.net.DFSNetworkTopology; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; @@ -61,11 +63,13 @@ import org.apache.hadoop.hdfs.server.protocol.SlowPeerReports; import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.apache.hadoop.net.DNSToSwitchMapping; +import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.util.Shell; import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; import org.mockito.internal.util.reflection.Whitebox; + import static org.hamcrest.core.Is.is; import static org.junit.Assert.*; @@ -622,4 +626,52 @@ public void testPendingRecoveryTasks() throws IOException { // Approximately load tasks if the ratio between queue length is large. verifyPendingRecoveryTasks(400, 1, 20, 20, 1); } + + @Test + public void testNetworkTopologyInstantiation() throws Exception { + // case 1, dfs.use.dfs.network.topology=true, use the default + // DFSNetworkTopology impl. + Configuration conf1 = new HdfsConfiguration(); + FSNamesystem fsn = Mockito.mock(FSNamesystem.class); + DatanodeManager dm1 = mockDatanodeManager(fsn, conf1); + assertEquals(DFSNetworkTopology.class, dm1.getNetworkTopology().getClass()); + + // case 2, dfs.use.dfs.network.topology=false, use the default + // NetworkTopology impl. + Configuration conf2 = new HdfsConfiguration(); + conf2.setBoolean(DFSConfigKeys.DFS_USE_DFS_NETWORK_TOPOLOGY_KEY, false); + DatanodeManager dm2 = mockDatanodeManager(fsn, conf2); + assertEquals(NetworkTopology.class, dm2.getNetworkTopology() + .getClass()); + + // case 3, dfs.use.dfs.network.topology=false, and specify the + // net.topology.impl property. + Configuration conf3 = new HdfsConfiguration(); + conf3.setClass(CommonConfigurationKeysPublic.NET_TOPOLOGY_IMPL_KEY, + MockDfsNetworkTopology.class, NetworkTopology.class); + conf3.setBoolean(DFSConfigKeys.DFS_USE_DFS_NETWORK_TOPOLOGY_KEY, false); + DatanodeManager dm3 = mockDatanodeManager(fsn, conf3); + assertEquals(MockDfsNetworkTopology.class, dm3.getNetworkTopology() + .getClass()); + + // case 4, dfs.use.dfs.network.topology=true, and specify the + // dfs.net.topology.impl property. + Configuration conf4 = new HdfsConfiguration(); + conf4.setClass(DFSConfigKeys.DFS_NET_TOPOLOGY_IMPL_KEY, + MockDfsNetworkTopology.class, NetworkTopology.class); + conf4.setBoolean(DFSConfigKeys.DFS_USE_DFS_NETWORK_TOPOLOGY_KEY, true); + DatanodeManager dm4 = mockDatanodeManager(fsn, conf4); + assertEquals(MockDfsNetworkTopology.class, dm4.getNetworkTopology() + .getClass()); + } + + /** + * A NetworkTopology implementation for test. + * + */ + public static class MockDfsNetworkTopology extends DFSNetworkTopology { + public MockDfsNetworkTopology(){ + super(); + } + } }