big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-17135. Update fsck -blockId to display excess state info of blocks (#5904)

Browse Source 
Reviewed-by: He Xiaoqiao <hexiaoqiao@apache.org>
Reviewed-by: Shilun Fan <slfan1989@apache.org>
Signed-off-by: Tao Li <tomscut@apache.org>
This commit is contained in:
huhaiyang 2023-07-31 09:21:22 +08:00 committed by GitHub
parent 688f76a822
commit 6cc56d40f5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 51 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
View File

@ -127,6 +127,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
"is ENTERING MAINTENANCE";
public static final String IN_MAINTENANCE_STATUS = "is IN MAINTENANCE";
public static final String STALE_STATUS = "is STALE";
public static final String EXCESS_STATUS = "is EXCESS";
public static final String NONEXISTENT_STATUS = "does not exist";
public static final String FAILURE_STATUS = "FAILED";
public static final String UNDEFINED = "undefined";
@ -373,6 +374,8 @@ private void printDatanodeReplicaStatus(Block block,
out.print(IN_MAINTENANCE_STATUS);
} else if (dn.isStale(this.staleInterval)) {
out.print(STALE_STATUS);
} else if (blockManager.isExcess(dn, blockManager.getStoredBlock(block))) {
out.print(EXCESS_STATUS);
} else {
out.print(HEALTHY_STATUS);
}

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
View File

@ -396,6 +396,8 @@ else if (args[idx].equals("-replicaDetails")) {
errCode = 0;
} else if (lastLine.contains("Incorrect blockId format:")) {
errCode = 0;
} else if (lastLine.endsWith(NamenodeFsck.EXCESS_STATUS)) {
errCode = 0;
} else if (lastLine.endsWith(NamenodeFsck.DECOMMISSIONED_STATUS)) {
errCode = 2;
} else if (lastLine.endsWith(NamenodeFsck.DECOMMISSIONING_STATUS)) {

49
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
View File

@ -1056,8 +1056,6 @@ public Boolean get() {
assertTrue(fsckOut.contains("(DECOMMISSIONED)"));
assertFalse(fsckOut.contains("(ENTERING MAINTENANCE)"));
assertFalse(fsckOut.contains("(IN MAINTENANCE)"));
}
/** Test if fsck can return -1 in case of failure.
@ -1834,6 +1832,51 @@ public void testBlockIdCKCorruption() throws Exception {
assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS));
}
/**
* Test for blockIdCK with block excess.
*/
@Test
public void testBlockIdCKExcess() throws Exception {
final Configuration configuration = new Configuration();
// Disable redundancy monitor check so that excess block can be verified.
configuration.setLong(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 5000);
try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(configuration).
numDataNodes(2).build()) {
cluster.waitActive();
final DistributedFileSystem fs = cluster.getFileSystem();
// Create file.
Path file = new Path("/test");
long fileLength = 512;
DFSTestUtil.createFile(fs, file, fileLength, (short) 2, 0L);
DFSTestUtil.waitReplication(fs, file, (short) 2);
List<LocatedBlock> locatedBlocks = DFSTestUtil.getAllBlocks(fs, file);
assertEquals(1, locatedBlocks.size());
String blockName = locatedBlocks.get(0).getBlock().getBlockName();
// Validate block is HEALTHY.
String outStr = runFsck(configuration, 0, true,
"/", "-blockId", blockName);
assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
assertTrue(outStr.contains("No. of Expected Replica: " + 2));
assertTrue(outStr.contains("No. of live Replica: " + 2));
assertTrue(outStr.contains("No. of excess Replica: " + 0));
// Make the block on one datanode enter excess state.
fs.setReplication(file, (short)1);
// Validate the one block is EXCESS.
outStr = runFsck(configuration, 0, true,
"/", "-blockId", blockName);
assertTrue(outStr.contains(NamenodeFsck.EXCESS_STATUS));
assertTrue(outStr.contains("No. of Expected Replica: " + 1));
assertTrue(outStr.contains("No. of live Replica: " + 1));
assertTrue(outStr.contains("No. of excess Replica: " + 1));
}
}
private void writeFile(final DistributedFileSystem dfs,
Path dir, String fileName) throws IOException {
Path filePath = new Path(dir.toString() + Path.SEPARATOR + fileName);
@ -1961,7 +2004,7 @@ public void testFsckWithDecommissionedReplicas() throws Exception {
// check the replica status should be healthy(0) after decommission
// is done
String fsckOut = runFsck(conf, 0, true, testFile);
runFsck(conf, 0, true, testFile);
}
/**