big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-16163. Avoid locking entire blockPinningFailures map (#3296)

Browse Source
This commit is contained in:
Viraj Jasani 2021-08-16 11:10:05 +05:30 committed by GitHub
parent 3aaac8a1f6
commit 919a99bbb4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 14 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
View File

@ -40,6 +40,7 @@
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@ -405,7 +406,8 @@ private void dispatch() {
// Pinned block can't be moved. Add this block into failure list.
// Later in the next iteration mover will exclude these blocks from
// pending moves.
target.getDDatanode().addBlockPinningFailures(this);
target.getDDatanode().addBlockPinningFailures(
this.reportedBlock.getBlock().getBlockId(), this.getSource());
return;
}
@ -643,7 +645,8 @@ public boolean equals(Object obj) {
/** blocks being moved but not confirmed yet */
private final List<PendingMove> pendings;
private volatile boolean hasFailure = false;
private Map<Long, Set<DatanodeInfo>> blockPinningFailures = new HashMap<>();
private final Map<Long, Set<DatanodeInfo>> blockPinningFailures =
new ConcurrentHashMap<>();
private volatile boolean hasSuccess = false;
private ExecutorService moveExecutor;
@ -729,16 +732,17 @@ void setHasFailure() {
this.hasFailure = true;
}
void addBlockPinningFailures(PendingMove pendingBlock) {
synchronized (blockPinningFailures) {
long blockId = pendingBlock.reportedBlock.getBlock().getBlockId();
Set<DatanodeInfo> pinnedLocations = blockPinningFailures.get(blockId);
private void addBlockPinningFailures(long blockId, DatanodeInfo source) {
blockPinningFailures.compute(blockId, (key, pinnedLocations) -> {
Set<DatanodeInfo> newPinnedLocations;
if (pinnedLocations == null) {
pinnedLocations = new HashSet<>();
blockPinningFailures.put(blockId, pinnedLocations);
newPinnedLocations = new HashSet<>();
} else {
newPinnedLocations = pinnedLocations;
}
pinnedLocations.add(pendingBlock.getSource());
}
newPinnedLocations.add(source);
return newPinnedLocations;
});
}
Map<Long, Set<DatanodeInfo>> getBlockPinningFailureList() {