diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index f140f08bb2..6caf1e7167 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -2415,8 +2415,14 @@ private void handleFileStat(LocatedFileStatus stat) throws IOException { if (stat.isFile()) { // file curFile = stat; } else if (recursive) { // directory - itors.push(curItor); - curItor = listLocatedStatus(stat.getPath()); + try { + RemoteIterator newDirItor = listLocatedStatus(stat.getPath()); + itors.push(curItor); + curItor = newDirItor; + } catch (FileNotFoundException ignored) { + LOGGER.debug("Directory {} deleted while attempting for recursive listing", + stat.getPath()); + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java index c4c5e17f97..2092231177 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java @@ -20,6 +20,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeys.FS_CLIENT_TOPOLOGY_RESOLUTION_ENABLED; import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_CONTEXT; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; @@ -28,6 +29,7 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import java.io.File; import java.io.FileNotFoundException; @@ -118,9 +120,11 @@ import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.Time; import org.apache.hadoop.util.concurrent.HadoopExecutors; +import org.apache.hadoop.util.functional.RemoteIterators; import org.junit.Assert; import org.junit.Test; import org.mockito.InOrder; +import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.event.Level; @@ -1552,6 +1556,56 @@ public void testListFiles() throws IOException { } } + @Test + public void testListFilesRecursive() throws IOException { + Configuration conf = getTestConfiguration(); + + try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();) { + DistributedFileSystem fs = cluster.getFileSystem(); + + // Create some directories and files. + Path dir = new Path("/dir"); + Path subDir1 = fs.makeQualified(new Path(dir, "subDir1")); + Path subDir2 = fs.makeQualified(new Path(dir, "subDir2")); + + fs.create(new Path(dir, "foo1")).close(); + fs.create(new Path(dir, "foo2")).close(); + fs.create(new Path(subDir1, "foo3")).close(); + fs.create(new Path(subDir2, "foo4")).close(); + + // Mock the filesystem, and throw FNF when listing is triggered for the subdirectory. + FileSystem mockFs = spy(fs); + Mockito.doThrow(new FileNotFoundException("")).when(mockFs).listLocatedStatus(eq(subDir1)); + List str = RemoteIterators.toList(mockFs.listFiles(dir, true)); + assertThat(str).hasSize(3); + + // Mock the filesystem to depict a scenario where the directory got deleted and a file + // got created with the same name. + Mockito.doReturn(getMockedIterator(subDir1)).when(mockFs).listLocatedStatus(eq(subDir1)); + + str = RemoteIterators.toList(mockFs.listFiles(dir, true)); + assertThat(str).hasSize(4); + } + } + + private static RemoteIterator getMockedIterator(Path subDir1) { + return new RemoteIterator() { + private int remainingEntries = 1; + + @Override + public boolean hasNext() throws IOException { + return remainingEntries > 0; + } + + @Override + public LocatedFileStatus next() throws IOException { + remainingEntries--; + return new LocatedFileStatus(0, false, 1, 1024, 0L, 0, null, null, null, null, subDir1, + false, false, false, null); + } + }; + } + @Test public void testListStatusOfSnapshotDirs() throws IOException { MiniDFSCluster cluster = new MiniDFSCluster.Builder(getTestConfiguration())