big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-13418. NetworkTopology should be configurable when enable DFSNetworkTopology. Contributed by Tao Jie.

Browse Source
This commit is contained in:
Yiqun Lin 2018-04-13 17:55:45 +08:00
parent 1a407bc990
commit 0725953efe
4 changed files with 81 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
View File

@ -21,6 +21,7 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.net.DFSNetworkTopology;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyRackFaultTolerant;
@ -1177,6 +1178,12 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
"dfs.use.dfs.network.topology";
public static final boolean DFS_USE_DFS_NETWORK_TOPOLOGY_DEFAULT = true;
public static final String DFS_NET_TOPOLOGY_IMPL_KEY =
"dfs.net.topology.impl";
public static final Class<DFSNetworkTopology> DFS_NET_TOPOLOGY_IMPL_DEFAULT =
DFSNetworkTopology.class;
// dfs.client.retry confs are moved to HdfsClientConfigKeys.Retry
@Deprecated
public static final String DFS_CLIENT_RETRY_POLICY_ENABLED_KEY

10
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/net/DFSNetworkTopology.java
View File

@ -22,11 +22,13 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.net.NodeBase;
import org.apache.hadoop.util.ReflectionUtils;
import java.util.ArrayList;
import java.util.Collection;
@ -44,8 +46,12 @@ public class DFSNetworkTopology extends NetworkTopology {
private static final Random RANDOM = new Random();
public static DFSNetworkTopology getInstance(Configuration conf) {
DFSNetworkTopology nt = new DFSNetworkTopology();
return (DFSNetworkTopology)nt.init(DFSTopologyNodeImpl.FACTORY);
DFSNetworkTopology nt = ReflectionUtils.newInstance(conf.getClass(
DFSConfigKeys.DFS_NET_TOPOLOGY_IMPL_KEY,
DFSConfigKeys.DFS_NET_TOPOLOGY_IMPL_DEFAULT,
DFSNetworkTopology.class), conf);
return (DFSNetworkTopology) nt.init(DFSTopologyNodeImpl.FACTORY);
}
/**

14
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
View File

@ -4859,6 +4859,20 @@
<value>true</value>
<description>
Enables DFSNetworkTopology to choose nodes for placing replicas.
When enabled, NetworkTopology will be instantiated as class defined in
property dfs.net.topology.impl, otherwise NetworkTopology will be
instantiated as class defined in property net.topology.impl.
</description>
</property>
<property>
<name>dfs.net.topology.impl</name>
<value>org.apache.hadoop.hdfs.net.DFSNetworkTopology</value>
<description>
The implementation class of NetworkTopology used in HDFS. By default,
the class org.apache.hadoop.hdfs.net.DFSNetworkTopology is specified and
used in block placement.
This property only works when dfs.use.dfs.network.topology is true.
</description>
</property>

52
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
View File

@ -41,6 +41,8 @@
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.net.DFSNetworkTopology;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
@ -61,11 +63,13 @@
import org.apache.hadoop.hdfs.server.protocol.SlowPeerReports;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
import org.apache.hadoop.net.DNSToSwitchMapping;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.util.Shell;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
import org.mockito.internal.util.reflection.Whitebox;
import static org.hamcrest.core.Is.is;
import static org.junit.Assert.*;
@ -622,4 +626,52 @@ public void testPendingRecoveryTasks() throws IOException {
// Approximately load tasks if the ratio between queue length is large.
verifyPendingRecoveryTasks(400, 1, 20, 20, 1);
}
@Test
public void testNetworkTopologyInstantiation() throws Exception {
// case 1, dfs.use.dfs.network.topology=true, use the default
// DFSNetworkTopology impl.
Configuration conf1 = new HdfsConfiguration();
FSNamesystem fsn = Mockito.mock(FSNamesystem.class);
DatanodeManager dm1 = mockDatanodeManager(fsn, conf1);
assertEquals(DFSNetworkTopology.class, dm1.getNetworkTopology().getClass());
// case 2, dfs.use.dfs.network.topology=false, use the default
// NetworkTopology impl.
Configuration conf2 = new HdfsConfiguration();
conf2.setBoolean(DFSConfigKeys.DFS_USE_DFS_NETWORK_TOPOLOGY_KEY, false);
DatanodeManager dm2 = mockDatanodeManager(fsn, conf2);
assertEquals(NetworkTopology.class, dm2.getNetworkTopology()
.getClass());
// case 3, dfs.use.dfs.network.topology=false, and specify the
// net.topology.impl property.
Configuration conf3 = new HdfsConfiguration();
conf3.setClass(CommonConfigurationKeysPublic.NET_TOPOLOGY_IMPL_KEY,
MockDfsNetworkTopology.class, NetworkTopology.class);
conf3.setBoolean(DFSConfigKeys.DFS_USE_DFS_NETWORK_TOPOLOGY_KEY, false);
DatanodeManager dm3 = mockDatanodeManager(fsn, conf3);
assertEquals(MockDfsNetworkTopology.class, dm3.getNetworkTopology()
.getClass());
// case 4, dfs.use.dfs.network.topology=true, and specify the
// dfs.net.topology.impl property.
Configuration conf4 = new HdfsConfiguration();
conf4.setClass(DFSConfigKeys.DFS_NET_TOPOLOGY_IMPL_KEY,
MockDfsNetworkTopology.class, NetworkTopology.class);
conf4.setBoolean(DFSConfigKeys.DFS_USE_DFS_NETWORK_TOPOLOGY_KEY, true);
DatanodeManager dm4 = mockDatanodeManager(fsn, conf4);
assertEquals(MockDfsNetworkTopology.class, dm4.getNetworkTopology()
.getClass());
}
/**
* A NetworkTopology implementation for test.
*
*/
public static class MockDfsNetworkTopology extends DFSNetworkTopology {
public MockDfsNetworkTopology(){
super();
}
}
}