big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-18662. ListFiles with recursive fails with FNF. (#5477). Contributed by Ayush Saxena.

Browse Source 
Reviewed-by: Steve Loughran <stevel@apache.org
This commit is contained in:
Ayush Saxena 2023-03-23 08:30:08 +05:30 committed by GitHub
parent 67e02a92e0
commit e3cb9573e1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 62 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
View File

@ -2413,8 +2413,14 @@ private void handleFileStat(LocatedFileStatus stat) throws IOException {
if (stat.isFile()) { // file
curFile = stat;
} else if (recursive) { // directory
itors.push(curItor);
curItor = listLocatedStatus(stat.getPath());
try {
RemoteIterator<LocatedFileStatus> newDirItor = listLocatedStatus(stat.getPath());
itors.push(curItor);
curItor = newDirItor;
} catch (FileNotFoundException ignored) {
LOGGER.debug("Directory {} deleted while attempting for recursive listing",
stat.getPath());
}
}
}

54
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
View File

@ -22,6 +22,7 @@
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_FILE_CLOSE_NUM_COMMITTED_ALLOWED_KEY;
import static org.apache.hadoop.hdfs.client.HdfsAdmin.TRASH_PERMISSION;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_CONTEXT;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@ -31,6 +32,7 @@
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import java.io.File;
import java.io.FileNotFoundException;
@ -123,9 +125,11 @@
import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.concurrent.HadoopExecutors;
import org.apache.hadoop.util.functional.RemoteIterators;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.InOrder;
import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.event.Level;
@ -1557,6 +1561,56 @@ public void testListFiles() throws IOException {
}
}
@Test
public void testListFilesRecursive() throws IOException {
Configuration conf = getTestConfiguration();
try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();) {
DistributedFileSystem fs = cluster.getFileSystem();
// Create some directories and files.
Path dir = new Path("/dir");
Path subDir1 = fs.makeQualified(new Path(dir, "subDir1"));
Path subDir2 = fs.makeQualified(new Path(dir, "subDir2"));
fs.create(new Path(dir, "foo1")).close();
fs.create(new Path(dir, "foo2")).close();
fs.create(new Path(subDir1, "foo3")).close();
fs.create(new Path(subDir2, "foo4")).close();
// Mock the filesystem, and throw FNF when listing is triggered for the subdirectory.
FileSystem mockFs = spy(fs);
Mockito.doThrow(new FileNotFoundException("")).when(mockFs).listLocatedStatus(eq(subDir1));
List<LocatedFileStatus> str = RemoteIterators.toList(mockFs.listFiles(dir, true));
assertThat(str).hasSize(3);
// Mock the filesystem to depict a scenario where the directory got deleted and a file
// got created with the same name.
Mockito.doReturn(getMockedIterator(subDir1)).when(mockFs).listLocatedStatus(eq(subDir1));
str = RemoteIterators.toList(mockFs.listFiles(dir, true));
assertThat(str).hasSize(4);
}
}
private static RemoteIterator<LocatedFileStatus> getMockedIterator(Path subDir1) {
return new RemoteIterator<LocatedFileStatus>() {
private int remainingEntries = 1;
@Override
public boolean hasNext() throws IOException {
return remainingEntries > 0;
}
@Override
public LocatedFileStatus next() throws IOException {
remainingEntries--;
return new LocatedFileStatus(0, false, 1, 1024, 0L, 0, null, null, null, null, subDir1,
false, false, false, null);
}
};
}
@Test
public void testListStatusOfSnapshotDirs() throws IOException {
MiniDFSCluster cluster = new MiniDFSCluster.Builder(getTestConfiguration())