big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-16879. EC: Fsck -blockId shows number of redundant internal block replicas for EC Blocks (#5264)

Browse Source
This commit is contained in:
huhaiyang 2023-01-04 11:38:32 +08:00 committed by GitHub
parent 0926fa5a2c
commit 35ce60eadd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 66 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
View File

@ -320,6 +320,10 @@ public void blockIdCK(String blockId) {
}
out.println("No. of corrupted Replica: " +
numberReplicas.corruptReplicas());
// for striped blocks only and number of redundant internal block replicas.
if (blockInfo.isStriped()) {
out.println("No. of redundant Replica: " + numberReplicas.redundantInternalBlocks());
}
//record datanodes that have corrupted block replica
Collection<DatanodeDescriptor> corruptionRecord = null;
if (blockManager.getCorruptReplicas(block) != null) {

62
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
View File

@ -2450,6 +2450,68 @@ public void testFsckMissingECFile() throws Exception {
assertTrue(outStr.contains("has 1 CORRUPT blocks"));
}
@Test
public void testFsckECBlockIdRedundantInternalBlocks() throws Exception {
final int dataBlocks = StripedFileTestUtil.getDefaultECPolicy().getNumDataUnits();
final int parityBlocks = StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits();
final int cellSize = StripedFileTestUtil.getDefaultECPolicy().getCellSize();
final short groupSize = (short) (dataBlocks + parityBlocks);
final File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
final Path dirPath = new Path("/ec_dir");
final Path filePath = new Path(dirPath, "file");
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
cluster = new MiniDFSCluster.Builder(conf, builderBaseDir).numDataNodes(groupSize + 1).build();
cluster.waitActive();
DistributedFileSystem fs = cluster.getFileSystem();
fs.enableErasureCodingPolicy(
StripedFileTestUtil.getDefaultECPolicy().getName());
try {
fs.mkdirs(dirPath);
fs.setErasureCodingPolicy(dirPath, StripedFileTestUtil.getDefaultECPolicy().getName());
DFSTestUtil.createFile(fs, filePath, cellSize * dataBlocks * 2, (short) 1, 0L);
LocatedBlocks blks = fs.getClient().getLocatedBlocks(filePath.toString(), 0);
LocatedStripedBlock block = (LocatedStripedBlock) blks.getLastLocatedBlock();
Assert.assertEquals(groupSize, block.getLocations().length);
//general test.
String runFsckResult = runFsck(conf, 0, true, "/",
"-blockId", block.getBlock().getBlockName());
assertTrue(runFsckResult.contains(block.getBlock().getBlockName()));
assertTrue(runFsckResult.contains("No. of Expected Replica: " + groupSize));
assertTrue(runFsckResult.contains("No. of live Replica: " + groupSize));
assertTrue(runFsckResult.contains("No. of redundant Replica: " + 0));
// stop a dn.
DatanodeInfo dnToStop = block.getLocations()[0];
MiniDFSCluster.DataNodeProperties dnProp = cluster.stopDataNode(dnToStop.getXferAddr());
cluster.setDataNodeDead(dnToStop);
// wait for reconstruction to happen.
DFSTestUtil.waitForReplication(fs, filePath, groupSize, 15 * 1000);
// bring the dn back: 10 internal blocks now.
cluster.restartDataNode(dnProp);
cluster.waitActive();
blks = fs.getClient().getLocatedBlocks(filePath.toString(), 0);
block = (LocatedStripedBlock) blks.getLastLocatedBlock();
Assert.assertEquals(groupSize + 1, block.getLocations().length);
//general test, number of redundant internal block replicas.
runFsckResult = runFsck(conf, 0, true, "/",
"-blockId", block.getBlock().getBlockName());
assertTrue(runFsckResult.contains(block.getBlock().getBlockName()));
assertTrue(runFsckResult.contains("No. of Expected Replica: " + groupSize));
assertTrue(runFsckResult.contains("No. of live Replica: " + groupSize));
assertTrue(runFsckResult.contains("No. of redundant Replica: " + 1));
} finally {
cluster.shutdown();
}
}
private void waitForUnrecoverableBlockGroup(Configuration configuration)
throws TimeoutException, InterruptedException {
GenericTestUtils.waitFor(new Supplier<Boolean>() {