HDFS-16676. DatanodeAdminManager$Monitor reports a node as invalid continuously (#4626)

Co-authored-by: Ashutosh Gupta <ashugpt@amazon.com>

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java

@@ -201,6 +201,7 @@ private void check() {
         iterkey).iterator();
     final List<DatanodeDescriptor> toRemove = new ArrayList<>();
     final List<DatanodeDescriptor> unhealthyDns = new ArrayList<>();
+    boolean isValidState = true;
 
     while (it.hasNext() && !exceededNumBlocksPerCheck() && namesystem
         .isRunning()) {
@@ -265,6 +266,7 @@ private void check() {
               // to track maintenance expiration.
               dnAdmin.setInMaintenance(dn);
             } else {
+              isValidState  = false;
               Preconditions.checkState(false,
                   "Node %s is in an invalid state! "
                       + "Invalid state: %s %s blocks are on this dn.",
@@ -288,7 +290,11 @@ private void check() {
         // an invalid state.
         LOG.warn("DatanodeAdminMonitor caught exception when processing node "
             + "{}.", dn, e);
+        if(isValidState){
           getPendingNodes().add(dn);
+        } else {
+          LOG.warn("Ignoring the node {} which is in invalid state", dn);
+        }
         toRemove.add(dn);
         unhealthyDns.remove(dn);
       } finally {