diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md index b464941537..b56666c4a2 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md @@ -1185,6 +1185,10 @@ on (possibly remote) filesystems. These filesystems are invariably accessed concurrently; the state of the filesystem MAY change between a `hasNext()` probe and the invocation of the `next()` call. +During iteration through a `RemoteIterator`, if the directory is deleted on +remote filesystem, then `hasNext()` or `next()` call may throw +`FileNotFoundException`. + Accordingly, a robust iteration through a `RemoteIterator` would catch and discard `NoSuchElementException` exceptions raised during the process, which could be done through the `while(true)` iteration example above, or diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java index 645f1ad833..cd870ca5ae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java @@ -232,7 +232,7 @@ public boolean hasNext() throws IOException { thisListing = dfs.listPaths(src, thisListing.getLastName(), needLocation); if (thisListing == null) { - return false; // the directory is deleted + throw new FileNotFoundException("File " + src + " does not exist."); } i = 0; } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 3e098045a7..f8af4abff4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -1168,7 +1168,7 @@ private boolean hasNextNoFilter() throws IOException { needLocation); statistics.incrementReadOps(1); if (thisListing == null) { - return false; + throw new FileNotFoundException("File " + p + " does not exist."); } i = 0; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java index c74bb632a7..31007ddb01 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java @@ -317,8 +317,31 @@ public void testGetFileStatusOnDir() throws Exception { assertEquals(file3.toString(), itor.next().getPath().toString()); assertFalse(itor.hasNext()); - - fs.delete(dir, true); + itor = fs.listStatusIterator(dir); + assertEquals(dir3.toString(), itor.next().getPath().toString()); + assertEquals(dir4.toString(), itor.next().getPath().toString()); + fs.delete(dir.getParent(), true); + try { + itor.hasNext(); + fail("FileNotFoundException expected"); + } catch (FileNotFoundException fnfe) { + } + + fs.mkdirs(file2); + fs.mkdirs(dir3); + fs.mkdirs(dir4); + fs.mkdirs(dir5); + itor = fs.listStatusIterator(dir); + int count = 0; + try { + fs.delete(dir.getParent(), true); + while (itor.next() != null) { + count++; + } + fail("FileNotFoundException expected"); + } catch (FileNotFoundException fnfe) { + } + assertEquals(2, count); } }