HDFS-17135. Update fsck -blockId to display excess state info of blocks (#5904)

Reviewed-by: He Xiaoqiao <hexiaoqiao@apache.org>
Reviewed-by: Shilun Fan <slfan1989@apache.org>
Signed-off-by: Tao Li <tomscut@apache.org>

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java

@@ -127,6 +127,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
       "is ENTERING MAINTENANCE";
   public static final String IN_MAINTENANCE_STATUS = "is IN MAINTENANCE";
   public static final String STALE_STATUS = "is STALE";
+  public static final String EXCESS_STATUS = "is EXCESS";
   public static final String NONEXISTENT_STATUS = "does not exist";
   public static final String FAILURE_STATUS = "FAILED";
   public static final String UNDEFINED = "undefined";
@@ -373,6 +374,8 @@ private void printDatanodeReplicaStatus(Block block,
       out.print(IN_MAINTENANCE_STATUS);
     } else if (dn.isStale(this.staleInterval)) {
       out.print(STALE_STATUS);
+    } else if (blockManager.isExcess(dn, blockManager.getStoredBlock(block))) {
+      out.print(EXCESS_STATUS);
     } else {
       out.print(HEALTHY_STATUS);
     }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java

@@ -396,6 +396,8 @@ else if (args[idx].equals("-replicaDetails")) {
       errCode = 0;
     } else if (lastLine.contains("Incorrect blockId format:")) {
       errCode = 0;
+    } else if (lastLine.endsWith(NamenodeFsck.EXCESS_STATUS)) {
+      errCode = 0;
     } else if (lastLine.endsWith(NamenodeFsck.DECOMMISSIONED_STATUS)) {
       errCode = 2;
     } else if (lastLine.endsWith(NamenodeFsck.DECOMMISSIONING_STATUS)) {

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java

@@ -1056,8 +1056,6 @@ public Boolean get() {
     assertTrue(fsckOut.contains("(DECOMMISSIONED)"));
     assertFalse(fsckOut.contains("(ENTERING MAINTENANCE)"));
     assertFalse(fsckOut.contains("(IN MAINTENANCE)"));
-
-
   }
 
   /** Test if fsck can return -1 in case of failure.
@@ -1834,6 +1832,51 @@ public void testBlockIdCKCorruption() throws Exception {
     assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS));
   }
 
+  /**
+   * Test for blockIdCK with block excess.
+   */
+  @Test
+  public void testBlockIdCKExcess() throws Exception {
+    final Configuration configuration = new Configuration();
+    // Disable redundancy monitor check so that excess block can be verified.
+    configuration.setLong(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 5000);
+
+    try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(configuration).
+        numDataNodes(2).build()) {
+      cluster.waitActive();
+      final DistributedFileSystem fs = cluster.getFileSystem();
+
+      // Create file.
+      Path file = new Path("/test");
+      long fileLength = 512;
+      DFSTestUtil.createFile(fs, file, fileLength, (short) 2, 0L);
+      DFSTestUtil.waitReplication(fs, file, (short) 2);
+
+      List<LocatedBlock> locatedBlocks = DFSTestUtil.getAllBlocks(fs, file);
+      assertEquals(1, locatedBlocks.size());
+      String blockName = locatedBlocks.get(0).getBlock().getBlockName();
+
+      // Validate block is HEALTHY.
+      String outStr = runFsck(configuration, 0, true,
+          "/", "-blockId", blockName);
+      assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
+      assertTrue(outStr.contains("No. of Expected Replica: " + 2));
+      assertTrue(outStr.contains("No. of live Replica: " + 2));
+      assertTrue(outStr.contains("No. of excess Replica: " + 0));
+
+      // Make the block on one  datanode enter excess state.
+      fs.setReplication(file, (short)1);
+
+      // Validate the one block is EXCESS.
+      outStr = runFsck(configuration, 0, true,
+          "/", "-blockId", blockName);
+      assertTrue(outStr.contains(NamenodeFsck.EXCESS_STATUS));
+      assertTrue(outStr.contains("No. of Expected Replica: " + 1));
+      assertTrue(outStr.contains("No. of live Replica: " + 1));
+      assertTrue(outStr.contains("No. of excess Replica: " + 1));
+    }
+  }
+
   private void writeFile(final DistributedFileSystem dfs,
       Path dir, String fileName) throws IOException {
     Path filePath = new Path(dir.toString() + Path.SEPARATOR + fileName);
@@ -1961,7 +2004,7 @@ public void testFsckWithDecommissionedReplicas() throws Exception {
 
     // check the replica status should be healthy(0) after decommission
     // is done
-    String fsckOut = runFsck(conf, 0, true, testFile);
+    runFsck(conf, 0, true, testFile);
   }
 
   /**