diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java index dfdb12285f..bc6df0141c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java @@ -127,6 +127,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory { "is ENTERING MAINTENANCE"; public static final String IN_MAINTENANCE_STATUS = "is IN MAINTENANCE"; public static final String STALE_STATUS = "is STALE"; + public static final String EXCESS_STATUS = "is EXCESS"; public static final String NONEXISTENT_STATUS = "does not exist"; public static final String FAILURE_STATUS = "FAILED"; public static final String UNDEFINED = "undefined"; @@ -373,6 +374,8 @@ private void printDatanodeReplicaStatus(Block block, out.print(IN_MAINTENANCE_STATUS); } else if (dn.isStale(this.staleInterval)) { out.print(STALE_STATUS); + } else if (blockManager.isExcess(dn, blockManager.getStoredBlock(block))) { + out.print(EXCESS_STATUS); } else { out.print(HEALTHY_STATUS); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java index 6587ffcce3..9f0288ebf3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java @@ -396,6 +396,8 @@ else if (args[idx].equals("-replicaDetails")) { errCode = 0; } else if (lastLine.contains("Incorrect blockId format:")) { errCode = 0; + } else if (lastLine.endsWith(NamenodeFsck.EXCESS_STATUS)) { + errCode = 0; } else if (lastLine.endsWith(NamenodeFsck.DECOMMISSIONED_STATUS)) { errCode = 2; } else if (lastLine.endsWith(NamenodeFsck.DECOMMISSIONING_STATUS)) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java index 5db4e029cb..7d6e7980be 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java @@ -1056,8 +1056,6 @@ public Boolean get() { assertTrue(fsckOut.contains("(DECOMMISSIONED)")); assertFalse(fsckOut.contains("(ENTERING MAINTENANCE)")); assertFalse(fsckOut.contains("(IN MAINTENANCE)")); - - } /** Test if fsck can return -1 in case of failure. @@ -1834,6 +1832,51 @@ public void testBlockIdCKCorruption() throws Exception { assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS)); } + /** + * Test for blockIdCK with block excess. + */ + @Test + public void testBlockIdCKExcess() throws Exception { + final Configuration configuration = new Configuration(); + // Disable redundancy monitor check so that excess block can be verified. + configuration.setLong(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 5000); + + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(configuration). + numDataNodes(2).build()) { + cluster.waitActive(); + final DistributedFileSystem fs = cluster.getFileSystem(); + + // Create file. + Path file = new Path("/test"); + long fileLength = 512; + DFSTestUtil.createFile(fs, file, fileLength, (short) 2, 0L); + DFSTestUtil.waitReplication(fs, file, (short) 2); + + List locatedBlocks = DFSTestUtil.getAllBlocks(fs, file); + assertEquals(1, locatedBlocks.size()); + String blockName = locatedBlocks.get(0).getBlock().getBlockName(); + + // Validate block is HEALTHY. + String outStr = runFsck(configuration, 0, true, + "/", "-blockId", blockName); + assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS)); + assertTrue(outStr.contains("No. of Expected Replica: " + 2)); + assertTrue(outStr.contains("No. of live Replica: " + 2)); + assertTrue(outStr.contains("No. of excess Replica: " + 0)); + + // Make the block on one datanode enter excess state. + fs.setReplication(file, (short)1); + + // Validate the one block is EXCESS. + outStr = runFsck(configuration, 0, true, + "/", "-blockId", blockName); + assertTrue(outStr.contains(NamenodeFsck.EXCESS_STATUS)); + assertTrue(outStr.contains("No. of Expected Replica: " + 1)); + assertTrue(outStr.contains("No. of live Replica: " + 1)); + assertTrue(outStr.contains("No. of excess Replica: " + 1)); + } + } + private void writeFile(final DistributedFileSystem dfs, Path dir, String fileName) throws IOException { Path filePath = new Path(dir.toString() + Path.SEPARATOR + fileName); @@ -1961,7 +2004,7 @@ public void testFsckWithDecommissionedReplicas() throws Exception { // check the replica status should be healthy(0) after decommission // is done - String fsckOut = runFsck(conf, 0, true, testFile); + runFsck(conf, 0, true, testFile); } /**