HDFS-11067. DFS#listStatusIterator(..) should throw FileNotFoundException if the directory deleted before fetching next batch of entries. Contributed by Vinayakumar B.

hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md

@@ -1185,6 +1185,10 @@ on (possibly remote) filesystems. These filesystems are invariably accessed
 concurrently; the state of the filesystem MAY change between a `hasNext()`
 probe and the invocation of the `next()` call.
 
+During iteration through a `RemoteIterator`, if the directory is deleted on
+remote filesystem, then `hasNext()` or `next()` call may throw
+`FileNotFoundException`.
+
 Accordingly, a robust iteration through a `RemoteIterator` would catch and
 discard `NoSuchElementException` exceptions raised during the process, which
 could be done through the `while(true)` iteration example above, or

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java

@@ -232,7 +232,7 @@ public boolean hasNext() throws IOException {
         thisListing = dfs.listPaths(src, thisListing.getLastName(),
             needLocation);
         if (thisListing == null) {
-          return false; // the directory is deleted
+          throw new FileNotFoundException("File " + src + " does not exist.");
         }
         i = 0;
       }

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java

@@ -1168,7 +1168,7 @@ private boolean hasNextNoFilter() throws IOException {
             needLocation);
         statistics.incrementReadOps(1);
         if (thisListing == null) {
-          return false;
+          throw new FileNotFoundException("File " + p + " does not exist.");
         }
         i = 0;
       }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java

@@ -318,7 +318,30 @@ public void testGetFileStatusOnDir() throws Exception {
 
     assertFalse(itor.hasNext());
 
+    itor = fs.listStatusIterator(dir);
+    assertEquals(dir3.toString(), itor.next().getPath().toString());
+    assertEquals(dir4.toString(), itor.next().getPath().toString());
+    fs.delete(dir.getParent(), true);
+    try {
+      itor.hasNext();
+      fail("FileNotFoundException expected");
+    } catch (FileNotFoundException fnfe) {
+    }
 
-    fs.delete(dir, true);
+    fs.mkdirs(file2);
+    fs.mkdirs(dir3);
+    fs.mkdirs(dir4);
+    fs.mkdirs(dir5);
+    itor = fs.listStatusIterator(dir);
+    int count = 0;
+    try {
+      fs.delete(dir.getParent(), true);
+      while (itor.next() != null) {
+        count++;
+      }
+      fail("FileNotFoundException expected");
+    } catch (FileNotFoundException fnfe) {
+    }
+    assertEquals(2, count);
   }
 }