big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-17150. EC: Fix the bug of failed lease recovery. (#5937). Contributed by Shuyan Zhang.

Browse Source 
Reviewed-by: hfutatzhanghb <1036798979@qq.com>
Reviewed-by: Haiyang Hu <haiyang.hu@shopee.com>
Signed-off-by: He Xiaoqiao <hexiaoqiao@apache.org>
This commit is contained in:
zhangshuyan 2023-08-15 19:54:15 +08:00 committed by GitHub
parent 8d95c588d2
commit 655c3df050
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 19 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
View File

@ -111,6 +111,7 @@
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotDeletionGc; import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotDeletionGc;
import org.apache.hadoop.thirdparty.protobuf.ByteString; import org.apache.hadoop.thirdparty.protobuf.ByteString;
@ -3802,7 +3803,12 @@ boolean internalReleaseLease(Lease lease, String src, INodesInPath iip,
lastBlock.getBlockType()); lastBlock.getBlockType());
} }
if (uc.getNumExpectedLocations() == 0 && lastBlock.getNumBytes() == 0) { int minLocationsNum = 1;
if (lastBlock.isStriped()) {
minLocationsNum = ((BlockInfoStriped) lastBlock).getRealDataBlockNum();
}
if (uc.getNumExpectedLocations() < minLocationsNum &&
lastBlock.getNumBytes() == 0) {
// There is no datanode reported to this block. // There is no datanode reported to this block.
// may be client have crashed before writing data to pipeline. // may be client have crashed before writing data to pipeline.
// This blocks doesn't need any recovery. // This blocks doesn't need any recovery.
@ -3810,8 +3816,18 @@ boolean internalReleaseLease(Lease lease, String src, INodesInPath iip,
pendingFile.removeLastBlock(lastBlock); pendingFile.removeLastBlock(lastBlock);
finalizeINodeFileUnderConstruction(src, pendingFile, finalizeINodeFileUnderConstruction(src, pendingFile,
iip.getLatestSnapshotId(), false); iip.getLatestSnapshotId(), false);
NameNode.stateChangeLog.warn("BLOCK* internalReleaseLease: " if (uc.getNumExpectedLocations() == 0) {
+ "Removed empty last block and closed file " + src); // If uc.getNumExpectedLocations() is 0, regardless of whether it
// is a striped block or not, we should consider it as an empty block.
NameNode.stateChangeLog.warn("BLOCK* internalReleaseLease: "
+ "Removed empty last block and closed file " + src);
} else {
// If uc.getNumExpectedLocations() is greater than 0, it means that
// minLocationsNum must be greater than 1, so this must be a striped
// block.
NameNode.stateChangeLog.warn("BLOCK* internalReleaseLease: "
+ "Removed last unrecoverable block group and closed file " + src);
}
return true; return true;
} }
// Start recovery of the last block for this file // Start recovery of the last block for this file