big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-17154. EC: Fix bug in updateBlockForPipeline after failover. (#5941). Contributed by Shuyan Zhang.

Browse Source 
Reviewed-by: Haiyang Hu <haiyang.hu@shopee.com>
Signed-off-by: He Xiaoqiao <hexiaoqiao@apache.org>
This commit is contained in:
zhangshuyan 2023-08-16 20:29:19 +08:00 committed by GitHub
parent ea87aa2f5b
commit 10b1d7340b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 26 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedStripedBlock.java
View File

@ -36,7 +36,7 @@ public class LocatedStripedBlock extends LocatedBlock {
private static final byte[] EMPTY_INDICES = {};
private static final Token<BlockTokenIdentifier> EMPTY_TOKEN = new Token<>();
private final byte[] blockIndices;
private byte[] blockIndices;
private Token<BlockTokenIdentifier>[] blockTokens;
@SuppressWarnings({"unchecked"})
@ -72,6 +72,10 @@ public byte[] getBlockIndices() {
return this.blockIndices;
}
public void setBlockIndices(byte[] blockIndices) {
this.blockIndices = blockIndices;
}
@Override
public boolean isStriped() {
return true;

23
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
View File

@ -102,6 +102,7 @@
import org.apache.hadoop.hdfs.protocol.ECTopologyVerifierResult;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
import org.apache.hadoop.hdfs.protocol.SnapshotStatus;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KEY;
import static org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.*;
@ -5966,8 +5967,26 @@ LocatedBlock bumpBlockGenerationStamp(ExtendedBlock block,
block.setGenerationStamp(nextGenerationStamp(
blockManager.isLegacyBlock(block.getLocalBlock())));
locatedBlock = BlockManager.newLocatedBlock(
block, file.getLastBlock(), null, -1);
BlockInfo lastBlockInfo = file.getLastBlock();
locatedBlock = BlockManager.newLocatedBlock(block, lastBlockInfo,
null, -1);
if (lastBlockInfo.isStriped() &&
((BlockInfoStriped) lastBlockInfo).getTotalBlockNum() >
((LocatedStripedBlock) locatedBlock).getBlockIndices().length) {
// The location info in BlockUnderConstructionFeature may not be
// complete after a failover, so we just return all block tokens for a
// striped block. This will disrupt the correspondence between
// LocatedStripedBlock.blockIndices and LocatedStripedBlock.locs,
// which is not used in client side. The correspondence between
// LocatedStripedBlock.blockIndices and LocatedBlock.blockToken is
// ensured.
byte[] indices =
new byte[((BlockInfoStriped) lastBlockInfo).getTotalBlockNum()];
for (int i = 0; i < indices.length; ++i) {
indices[i] = (byte) i;
}
((LocatedStripedBlock) locatedBlock).setBlockIndices(indices);
}
blockManager.setBlockToken(locatedBlock,
BlockTokenIdentifier.AccessMode.WRITE);
} finally {