big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-15363. BlockPlacementPolicyWithNodeGroup should validate if it is initialized by NetworkTopologyWithNodeGroup. Contributed by hemanthboyina.

Browse Source 
(cherry picked from commit 4d22d1c58f)
This commit is contained in:
Takanobu Asanuma 2020-05-23 17:28:38 +09:00
parent 9ce913a7bf
commit c44d8b8bdf
2 changed files with 32 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java
View File

@ -46,6 +46,11 @@ protected BlockPlacementPolicyWithNodeGroup() {
public void initialize(Configuration conf, FSClusterStats stats,
NetworkTopology clusterMap,
Host2NodesMap host2datanodeMap) {
if (!(clusterMap instanceof NetworkTopologyWithNodeGroup)) {
throw new IllegalArgumentException(
"Configured cluster topology should be "
+ NetworkTopologyWithNodeGroup.class.getName());
}
super.initialize(conf, stats, clusterMap, host2datanodeMap);
}

27
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java
View File

@ -47,6 +47,7 @@
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyWithNodeGroup;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.net.NetworkTopologyWithNodeGroup;
import org.apache.hadoop.test.LambdaTestUtils;
import org.junit.Assert;
import org.junit.Test;
@ -361,4 +362,30 @@ public void testBalancerEndInNoMoveProgress() throws Exception {
cluster.shutdown();
}
}
/**
* verify BlockPlacementPolicyNodeGroup uses NetworkTopologyWithNodeGroup.
*/
@Test
public void testBPPNodeGroup() throws Exception {
Configuration conf = createConf();
conf.setBoolean(DFSConfigKeys.DFS_USE_DFS_NETWORK_TOPOLOGY_KEY, true);
long[] capacities = new long[] {CAPACITY, CAPACITY, CAPACITY, CAPACITY};
String[] racks = new String[] {RACK0, RACK0, RACK1, RACK1};
String[] nodeGroups =
new String[] {NODEGROUP0, NODEGROUP0, NODEGROUP1, NODEGROUP2};
int numOfDatanodes = capacities.length;
assertEquals(numOfDatanodes, racks.length);
assertEquals(numOfDatanodes, nodeGroups.length);
MiniDFSCluster.Builder builder =
new MiniDFSCluster.Builder(conf).numDataNodes(capacities.length)
.racks(racks).simulatedCapacities(capacities);
MiniDFSClusterWithNodeGroup.setNodeGroups(nodeGroups);
LambdaTestUtils.intercept(IllegalArgumentException.class,
"Configured cluster topology should be "
+ "org.apache.hadoop.net.NetworkTopologyWithNodeGroup",
() -> new MiniDFSClusterWithNodeGroup(builder));
}
}