big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-10453. ReplicationMonitor thread could stuck for long time due to the race between replication and delete of same file in a large cluster.. Contributed by He Xiaoqiao.

Browse Source
This commit is contained in:
Arpit Agarwal 2018-02-12 07:00:50 -08:00
parent 8cf88fcd1f
commit 96bb6a51ec
3 changed files with 9 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockReconstructionWork.java
View File

@ -33,6 +33,7 @@ abstract class BlockReconstructionWork {
private final BlockInfo block; private final BlockInfo block;
private final String srcPath; private final String srcPath;
private final long blockSize;
private final byte storagePolicyID; private final byte storagePolicyID;
/** /**
@ -59,6 +60,7 @@ public BlockReconstructionWork(BlockInfo block,
int priority) { int priority) {
this.block = block; this.block = block;
this.srcPath = bc.getName(); this.srcPath = bc.getName();
this.blockSize = block.getNumBytes();
this.storagePolicyID = bc.getStoragePolicyID(); this.storagePolicyID = bc.getStoragePolicyID();
this.srcNodes = srcNodes; this.srcNodes = srcNodes;
this.containingNodes = containingNodes; this.containingNodes = containingNodes;
@ -100,6 +102,10 @@ public String getSrcPath() {
return srcPath; return srcPath;
} }
public long getBlockSize() {
return blockSize;
}
public byte getStoragePolicyID() { public byte getStoragePolicyID() {
return storagePolicyID; return storagePolicyID;
} }

3
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java
View File

@ -59,8 +59,7 @@ void chooseTargets(BlockPlacementPolicy blockplacement,
// TODO: new placement policy for EC considering multiple writers // TODO: new placement policy for EC considering multiple writers
DatanodeStorageInfo[] chosenTargets = blockplacement.chooseTarget( DatanodeStorageInfo[] chosenTargets = blockplacement.chooseTarget(
getSrcPath(), getAdditionalReplRequired(), getSrcNodes()[0], getSrcPath(), getAdditionalReplRequired(), getSrcNodes()[0],
getLiveReplicaStorages(), false, excludedNodes, getLiveReplicaStorages(), false, excludedNodes, getBlockSize(),
getBlock().getNumBytes(),
storagePolicySuite.getPolicy(getStoragePolicyID()), null); storagePolicySuite.getPolicy(getStoragePolicyID()), null);
setTargets(chosenTargets); setTargets(chosenTargets);
} }

6
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicationWork.java
View File

@ -45,10 +45,8 @@ assert getSrcNodes().length > 0
try { try {
DatanodeStorageInfo[] chosenTargets = blockplacement.chooseTarget( DatanodeStorageInfo[] chosenTargets = blockplacement.chooseTarget(
getSrcPath(), getAdditionalReplRequired(), getSrcNodes()[0], getSrcPath(), getAdditionalReplRequired(), getSrcNodes()[0],
getLiveReplicaStorages(), false, excludedNodes, getLiveReplicaStorages(), false, excludedNodes, getBlockSize(),
getBlock().getNumBytes(), storagePolicySuite.getPolicy(getStoragePolicyID()), null);
storagePolicySuite.getPolicy(getStoragePolicyID()),
null);
setTargets(chosenTargets); setTargets(chosenTargets);
} finally { } finally {
getSrcNodes()[0].decrementPendingReplicationWithoutTargets(); getSrcNodes()[0].decrementPendingReplicationWithoutTargets();