big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-14102. Performance improvement in BlockPlacementPolicyDefault. Contributed by Beluga Behr.

Browse Source
This commit is contained in:
Giovanni Matteo Fumarola 2018-11-28 11:33:22 -08:00
parent 4ca3a6b21a
commit 300f560fcc
1 changed files with 13 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
View File

@ -71,6 +71,9 @@ protected StringBuilder initialValue() {
CHOOSE_RANDOM_REASONS = ThreadLocal CHOOSE_RANDOM_REASONS = ThreadLocal
.withInitial(() -> new HashMap<NodeNotChosenReason, Integer>()); .withInitial(() -> new HashMap<NodeNotChosenReason, Integer>());
private static final BlockPlacementStatus ONE_RACK_PLACEMENT =
new BlockPlacementStatusDefault(1, 1, 1);
private enum NodeNotChosenReason { private enum NodeNotChosenReason {
NOT_IN_SERVICE("the node is not in service"), NOT_IN_SERVICE("the node is not in service"),
NODE_STALE("the node is stale"), NODE_STALE("the node is stale"),
@ -1029,22 +1032,23 @@ private DatanodeStorageInfo[] getPipeline(Node writer,
@Override @Override
public BlockPlacementStatus verifyBlockPlacement(DatanodeInfo[] locs, public BlockPlacementStatus verifyBlockPlacement(DatanodeInfo[] locs,
int numberOfReplicas) { int numberOfReplicas) {
if (locs == null) if (locs == null) {
locs = DatanodeDescriptor.EMPTY_ARRAY; locs = DatanodeDescriptor.EMPTY_ARRAY;
}
if (!clusterMap.hasClusterEverBeenMultiRack()) { if (!clusterMap.hasClusterEverBeenMultiRack()) {
// only one rack // only one rack
return new BlockPlacementStatusDefault(1, 1, 1); return ONE_RACK_PLACEMENT;
} }
int minRacks = 2; final int minRacks = Math.min(2, numberOfReplicas);
minRacks = Math.min(minRacks, numberOfReplicas);
// 1. Check that all locations are different. // 1. Check that all locations are different.
// 2. Count locations on different racks. // 2. Count locations on different racks.
Set<String> racks = new TreeSet<>(); final long rackCount = Arrays.asList(locs).stream()
for (DatanodeInfo dn : locs) .map(dn -> dn.getNetworkLocation()).distinct().count();
racks.add(dn.getNetworkLocation());
return new BlockPlacementStatusDefault(racks.size(), minRacks, return new BlockPlacementStatusDefault(Math.toIntExact(rackCount),
clusterMap.getNumOfRacks()); minRacks, clusterMap.getNumOfRacks());
} }
/** /**
* Decide whether deleting the specified replica of the block still makes * Decide whether deleting the specified replica of the block still makes
* the block conform to the configured block placement policy. * the block conform to the configured block placement policy.