HADOOP-15637. LocalFs#listLocatedStatus does not filter out hidden .crc files. Contributed by Erik Krogen.
This commit is contained in:
parent
3517a47897
commit
e8f952ef06
@ -27,10 +27,12 @@
|
|||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.EnumSet;
|
import java.util.EnumSet;
|
||||||
|
|
||||||
|
import java.util.NoSuchElementException;
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.classification.InterfaceStability;
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
import org.apache.hadoop.fs.Options.ChecksumOpt;
|
import org.apache.hadoop.fs.Options.ChecksumOpt;
|
||||||
import org.apache.hadoop.fs.permission.FsPermission;
|
import org.apache.hadoop.fs.permission.FsPermission;
|
||||||
|
import org.apache.hadoop.security.AccessControlException;
|
||||||
import org.apache.hadoop.util.DataChecksum;
|
import org.apache.hadoop.util.DataChecksum;
|
||||||
import org.apache.hadoop.util.Progressable;
|
import org.apache.hadoop.util.Progressable;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
@ -527,4 +529,39 @@ public FileStatus[] listStatus(Path f) throws IOException,
|
|||||||
}
|
}
|
||||||
return results.toArray(new FileStatus[results.size()]);
|
return results.toArray(new FileStatus[results.size()]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f)
|
||||||
|
throws AccessControlException, FileNotFoundException,
|
||||||
|
UnresolvedLinkException, IOException {
|
||||||
|
final RemoteIterator<LocatedFileStatus> iter =
|
||||||
|
getMyFs().listLocatedStatus(f);
|
||||||
|
return new RemoteIterator<LocatedFileStatus>() {
|
||||||
|
|
||||||
|
private LocatedFileStatus next = null;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean hasNext() throws IOException {
|
||||||
|
while (next == null && iter.hasNext()) {
|
||||||
|
LocatedFileStatus unfilteredNext = iter.next();
|
||||||
|
if (!isChecksumFile(unfilteredNext.getPath())) {
|
||||||
|
next = unfilteredNext;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return next != null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public LocatedFileStatus next() throws IOException {
|
||||||
|
if (!hasNext()) {
|
||||||
|
throw new NoSuchElementException();
|
||||||
|
}
|
||||||
|
LocatedFileStatus tmp = next;
|
||||||
|
next = null;
|
||||||
|
return tmp;
|
||||||
|
}
|
||||||
|
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -369,6 +369,44 @@ public void testListStatus() throws Exception {
|
|||||||
pathsIterator = fc.listStatus(getTestRootPath(fc, "test/hadoop/a"));
|
pathsIterator = fc.listStatus(getTestRootPath(fc, "test/hadoop/a"));
|
||||||
Assert.assertFalse(pathsIterator.hasNext());
|
Assert.assertFalse(pathsIterator.hasNext());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testListFiles() throws Exception {
|
||||||
|
Path[] testDirs = {
|
||||||
|
getTestRootPath(fc, "test/dir1"),
|
||||||
|
getTestRootPath(fc, "test/dir1/dir1"),
|
||||||
|
getTestRootPath(fc, "test/dir2")
|
||||||
|
};
|
||||||
|
Path[] testFiles = {
|
||||||
|
new Path(testDirs[0], "file1"),
|
||||||
|
new Path(testDirs[0], "file2"),
|
||||||
|
new Path(testDirs[1], "file2"),
|
||||||
|
new Path(testDirs[2], "file1")
|
||||||
|
};
|
||||||
|
|
||||||
|
for (Path path : testDirs) {
|
||||||
|
fc.mkdir(path, FsPermission.getDefault(), true);
|
||||||
|
}
|
||||||
|
for (Path p : testFiles) {
|
||||||
|
FSDataOutputStream out = fc.create(p).build();
|
||||||
|
out.writeByte(0);
|
||||||
|
out.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
RemoteIterator<LocatedFileStatus> filesIterator =
|
||||||
|
fc.util().listFiles(getTestRootPath(fc, "test"), true);
|
||||||
|
LocatedFileStatus[] fileStats =
|
||||||
|
new LocatedFileStatus[testFiles.length];
|
||||||
|
for (int i = 0; i < fileStats.length; i++) {
|
||||||
|
assertTrue(filesIterator.hasNext());
|
||||||
|
fileStats[i] = filesIterator.next();
|
||||||
|
}
|
||||||
|
assertFalse(filesIterator.hasNext());
|
||||||
|
|
||||||
|
for (Path p : testFiles) {
|
||||||
|
assertTrue(containsPath(p, fileStats));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testListStatusFilterWithNoMatches() throws Exception {
|
public void testListStatusFilterWithNoMatches() throws Exception {
|
||||||
|
Loading…
Reference in New Issue
Block a user