big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-15637. LocalFs#listLocatedStatus does not filter out hidden .crc files. Contributed by Erik Krogen.

Browse Source
This commit is contained in:
Chen Liang 2018-07-30 10:25:07 -07:00
parent 3517a47897
commit e8f952ef06
2 changed files with 75 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
View File

@ -27,10 +27,12 @@
import java.util.Arrays;
import java.util.EnumSet;
import java.util.NoSuchElementException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.Progressable;
import org.slf4j.Logger;
@ -527,4 +529,39 @@ public FileStatus[] listStatus(Path f) throws IOException,
}
return results.toArray(new FileStatus[results.size()]);
}
@Override
public RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f)
throws AccessControlException, FileNotFoundException,
UnresolvedLinkException, IOException {
final RemoteIterator<LocatedFileStatus> iter =
getMyFs().listLocatedStatus(f);
return new RemoteIterator<LocatedFileStatus>() {
private LocatedFileStatus next = null;
@Override
public boolean hasNext() throws IOException {
while (next == null && iter.hasNext()) {
LocatedFileStatus unfilteredNext = iter.next();
if (!isChecksumFile(unfilteredNext.getPath())) {
next = unfilteredNext;
}
}
return next != null;
}
@Override
public LocatedFileStatus next() throws IOException {
if (!hasNext()) {
throw new NoSuchElementException();
}
LocatedFileStatus tmp = next;
next = null;
return tmp;
}
};
}
}

38
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
View File

@ -369,6 +369,44 @@ public void testListStatus() throws Exception {
pathsIterator = fc.listStatus(getTestRootPath(fc, "test/hadoop/a"));
Assert.assertFalse(pathsIterator.hasNext());
}
@Test
public void testListFiles() throws Exception {
Path[] testDirs = {
getTestRootPath(fc, "test/dir1"),
getTestRootPath(fc, "test/dir1/dir1"),
getTestRootPath(fc, "test/dir2")
};
Path[] testFiles = {
new Path(testDirs[0], "file1"),
new Path(testDirs[0], "file2"),
new Path(testDirs[1], "file2"),
new Path(testDirs[2], "file1")
};
for (Path path : testDirs) {
fc.mkdir(path, FsPermission.getDefault(), true);
}
for (Path p : testFiles) {
FSDataOutputStream out = fc.create(p).build();
out.writeByte(0);
out.close();
}
RemoteIterator<LocatedFileStatus> filesIterator =
fc.util().listFiles(getTestRootPath(fc, "test"), true);
LocatedFileStatus[] fileStats =
new LocatedFileStatus[testFiles.length];
for (int i = 0; i < fileStats.length; i++) {
assertTrue(filesIterator.hasNext());
fileStats[i] = filesIterator.next();
}
assertFalse(filesIterator.hasNext());
for (Path p : testFiles) {
assertTrue(containsPath(p, fileStats));
}
}
@Test
public void testListStatusFilterWithNoMatches() throws Exception {