big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-17177. ErasureCodingWork should ignore the deleted block while reconstructing blocks (#6024)

Browse Source
This commit is contained in:
huhaiyang 2023-09-11 11:57:52 +08:00 committed by GitHub
parent c5e9510b54
commit 3bd6a751ed
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 16 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ErasureCodingWork.java
View File

@ -62,10 +62,18 @@ void chooseTargets(BlockPlacementPolicy blockplacement,
BlockStoragePolicySuite storagePolicySuite, BlockStoragePolicySuite storagePolicySuite,
Set<Node> excludedNodes) { Set<Node> excludedNodes) {
// TODO: new placement policy for EC considering multiple writers // TODO: new placement policy for EC considering multiple writers
DatanodeStorageInfo[] chosenTargets = blockplacement.chooseTarget( DatanodeStorageInfo[] chosenTargets = null;
// HDFS-14720. If the block is deleted, the block size will become
// BlockCommand.NO_ACK (LONG.MAX_VALUE) . This kind of block we don't need
// to send for replication or reconstruction
if (!getBlock().isDeleted()) {
chosenTargets = blockplacement.chooseTarget(
getSrcPath(), getAdditionalReplRequired(), getSrcNodes()[0], getSrcPath(), getAdditionalReplRequired(), getSrcNodes()[0],
getLiveReplicaStorages(), false, excludedNodes, getBlockSize(), getLiveReplicaStorages(), false, excludedNodes, getBlockSize(),
storagePolicySuite.getPolicy(getStoragePolicyID()), null); storagePolicySuite.getPolicy(getStoragePolicyID()), null);
} else {
LOG.warn("ErasureCodingWork could not need choose targets for {}", getBlock());
}
setTargets(chosenTargets); setTargets(chosenTargets);
} }

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicationWork.java
View File

@ -17,7 +17,6 @@
*/ */
package org.apache.hadoop.hdfs.server.blockmanagement; package org.apache.hadoop.hdfs.server.blockmanagement;
import org.apache.hadoop.hdfs.server.protocol.BlockCommand;
import org.apache.hadoop.net.Node; import org.apache.hadoop.net.Node;
import java.util.List; import java.util.List;
@ -47,11 +46,13 @@ assert getSrcNodes().length > 0
// HDFS-14720 If the block is deleted, the block size will become // HDFS-14720 If the block is deleted, the block size will become
// BlockCommand.NO_ACK (LONG.MAX_VALUE) . This kind of block we don't need // BlockCommand.NO_ACK (LONG.MAX_VALUE) . This kind of block we don't need
// to send for replication or reconstruction // to send for replication or reconstruction
if (getBlock().getNumBytes() != BlockCommand.NO_ACK) { if (!getBlock().isDeleted()) {
chosenTargets = blockplacement.chooseTarget(getSrcPath(), chosenTargets = blockplacement.chooseTarget(getSrcPath(),
getAdditionalReplRequired(), getSrcNodes()[0], getAdditionalReplRequired(), getSrcNodes()[0],
getLiveReplicaStorages(), false, excludedNodes, getBlockSize(), getLiveReplicaStorages(), false, excludedNodes, getBlockSize(),
storagePolicySuite.getPolicy(getStoragePolicyID()), null); storagePolicySuite.getPolicy(getStoragePolicyID()), null);
} else {
LOG.warn("ReplicationWork could not need choose targets for {}", getBlock());
} }
setTargets(chosenTargets); setTargets(chosenTargets);
} finally { } finally {