big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-11295. Check storage remaining instead of node remaining in BlockPlacementPolicyDefault.chooseReplicaToDelete(). Contributed by Marton Elek.

Browse Source
This commit is contained in:
Arpit Agarwal 2017-02-24 15:44:11 -08:00
parent 289bc50e66
commit d2b3ba9b8f
4 changed files with 49 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
View File

@ -968,7 +968,7 @@ public DatanodeStorageInfo chooseReplicaToDelete(
} }
final DatanodeDescriptor node = storage.getDatanodeDescriptor(); final DatanodeDescriptor node = storage.getDatanodeDescriptor();
long free = node.getRemaining(); long free = storage.getRemaining();
long lastHeartbeat = node.getLastUpdateMonotonic(); long lastHeartbeat = node.getLastUpdateMonotonic();
if (lastHeartbeat < oldestHeartbeat) { if (lastHeartbeat < oldestHeartbeat) {
oldestHeartbeat = lastHeartbeat; oldestHeartbeat = lastHeartbeat;

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
View File

@ -388,6 +388,11 @@ static DatanodeStorageInfo getDatanodeStorageInfo(
return null; return null;
} }
@VisibleForTesting
void setRemainingForTests(int remaining) {
this.remaining = remaining;
}
static enum AddBlockResult { static enum AddBlockResult {
ADDED, REPLACED, ALREADY_EXIST ADDED, REPLACED, ALREADY_EXIST
} }

23
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
View File

@ -951,16 +951,23 @@ public void testChooseReplicaToDelete() throws Exception {
final Map<String, List<DatanodeStorageInfo>> rackMap final Map<String, List<DatanodeStorageInfo>> rackMap
= new HashMap<String, List<DatanodeStorageInfo>>(); = new HashMap<String, List<DatanodeStorageInfo>>();
dataNodes[0].setRemaining(4*1024*1024); storages[0].setRemainingForTests(4*1024*1024);
dataNodes[0].setRemaining(calculateRemaining(dataNodes[0]));
replicaList.add(storages[0]); replicaList.add(storages[0]);
dataNodes[1].setRemaining(3*1024*1024); storages[1].setRemainingForTests(3*1024*1024);
dataNodes[1].setRemaining(calculateRemaining(dataNodes[1]));
replicaList.add(storages[1]); replicaList.add(storages[1]);
dataNodes[2].setRemaining(2*1024*1024); storages[2].setRemainingForTests(2*1024*1024);
dataNodes[2].setRemaining(calculateRemaining(dataNodes[2]));
replicaList.add(storages[2]); replicaList.add(storages[2]);
dataNodes[5].setRemaining(1*1024*1024); //Even if this node has the most space, because the storage[5] has
//the lowest it should be chosen in case of block delete.
storages[4].setRemainingForTests(100 * 1024 * 1024);
storages[5].setRemainingForTests(512 * 1024);
dataNodes[5].setRemaining(calculateRemaining(dataNodes[5]));
replicaList.add(storages[5]); replicaList.add(storages[5]);
// Refresh the last update time for all the datanodes // Refresh the last update time for all the datanodes
@ -999,6 +1006,14 @@ public void testChooseReplicaToDelete() throws Exception {
assertEquals(chosen, storages[1]); assertEquals(chosen, storages[1]);
} }
private long calculateRemaining(DatanodeDescriptor dataNode) {
long sum = 0;
for (DatanodeStorageInfo storageInfo: dataNode.getStorageInfos()){
sum += storageInfo.getRemaining();
}
return sum;
}
@Test @Test
public void testChooseReplicasToDelete() throws Exception { public void testChooseReplicasToDelete() throws Exception {
Collection<DatanodeStorageInfo> nonExcess = new ArrayList<>(); Collection<DatanodeStorageInfo> nonExcess = new ArrayList<>();

21
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
View File

@ -625,16 +625,21 @@ public void testRereplicate3() throws Exception {
public void testChooseReplicaToDelete() throws Exception { public void testChooseReplicaToDelete() throws Exception {
List<DatanodeStorageInfo> replicaList = new ArrayList<>(); List<DatanodeStorageInfo> replicaList = new ArrayList<>();
final Map<String, List<DatanodeStorageInfo>> rackMap = new HashMap<>(); final Map<String, List<DatanodeStorageInfo>> rackMap = new HashMap<>();
dataNodes[0].setRemaining(4*1024*1024); storages[0].setRemainingForTests(4*1024*1024);
dataNodes[0].setRemaining(calculateRemaining(dataNodes[0]));
replicaList.add(storages[0]); replicaList.add(storages[0]);
dataNodes[1].setRemaining(3*1024*1024); storages[1].setRemainingForTests(3*1024*1024);
dataNodes[1].setRemaining(calculateRemaining(dataNodes[1]));
replicaList.add(storages[1]); replicaList.add(storages[1]);
dataNodes[2].setRemaining(2*1024*1024); storages[2].setRemainingForTests(2*1024*1024);
dataNodes[2].setRemaining(calculateRemaining(dataNodes[2]));
replicaList.add(storages[2]); replicaList.add(storages[2]);
dataNodes[5].setRemaining(1*1024*1024); storages[4].setRemainingForTests(100 * 1024 * 1024);
storages[5].setRemainingForTests(512 * 1024);
dataNodes[5].setRemaining(calculateRemaining(dataNodes[5]));
replicaList.add(storages[5]); replicaList.add(storages[5]);
List<DatanodeStorageInfo> first = new ArrayList<>(); List<DatanodeStorageInfo> first = new ArrayList<>();
@ -672,6 +677,14 @@ public void testChooseReplicaToDelete() throws Exception {
assertEquals(chosen, storages[5]); assertEquals(chosen, storages[5]);
} }
private long calculateRemaining(DatanodeDescriptor dataNode) {
long sum = 0;
for (DatanodeStorageInfo storageInfo: dataNode.getStorageInfos()){
sum += storageInfo.getRemaining();
}
return sum;
}
/** /**
* Test replica placement policy in case of boundary topology. * Test replica placement policy in case of boundary topology.
* Rack 2 has only 1 node group & can't be placed with two replicas * Rack 2 has only 1 node group & can't be placed with two replicas