big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-12169 ListStatus on empty dir in S3A lists itself instead of returning an empty list. author: Pieter Reuse.

Browse Source
This commit is contained in:
Steve Loughran 2016-04-03 16:39:14 +01:00
parent 54b2e78fd2
commit 0ecdd4cffa
3 changed files with 33 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetFileStatusTest.java
View File

@ -19,10 +19,14 @@
package org.apache.hadoop.fs.contract;
import java.io.FileNotFoundException;
import java.io.IOException;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -58,4 +62,23 @@ public void testGetFileStatusNonexistentFile() throws Throwable {
handleExpectedException(e);
}
}
@Test
public void testListStatusEmptyDirectory() throws IOException {
// remove the test directory
FileSystem fs = getFileSystem();
assertTrue(fs.delete(getContract().getTestPath(), true));
// create a - non-qualified - Path for a subdir
Path subfolder = getContract().getTestPath().suffix("/"+testPath.getName());
assertTrue(fs.mkdirs(subfolder));
// assert empty ls on the empty dir
assertEquals("ls on an empty directory not of length 0", 0,
fs.listStatus(subfolder).length);
// assert non-empty ls on parent dir
assertTrue("ls on a non-empty directory of length 0",
fs.listStatus(getContract().getTestPath()).length > 0);
}
}

7
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
View File

@ -788,11 +788,14 @@ public FileStatus[] listStatus(Path f) throws FileNotFoundException,
ObjectListing objects = s3.listObjects(request);
statistics.incrementReadOps(1);
Path fQualified = f.makeQualified(uri, workingDir);
while (true) {
for (S3ObjectSummary summary : objects.getObjectSummaries()) {
Path keyPath = keyToPath(summary.getKey()).makeQualified(uri, workingDir);
// Skip over keys that are ourselves and old S3N _$folder$ files
if (keyPath.equals(f) || summary.getKey().endsWith(S3N_FOLDER_SUFFIX)) {
if (keyPath.equals(fQualified) ||
summary.getKey().endsWith(S3N_FOLDER_SUFFIX)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Ignoring: " + keyPath);
}
@ -807,7 +810,7 @@ public FileStatus[] listStatus(Path f) throws FileNotFoundException,
} else {
result.add(new S3AFileStatus(summary.getSize(),
dateToLong(summary.getLastModified()), keyPath,
getDefaultBlockSize(f.makeQualified(uri, workingDir))));
getDefaultBlockSize(fQualified)));
if (LOG.isDebugEnabled()) {
LOG.debug("Adding: fi: " + keyPath);
}

5
hadoop-tools/hadoop-aws/src/test/resources/contract/s3a.xml
View File

@ -77,6 +77,11 @@
<value>false</value>
</property>
<property>
<name>fs.contract.supports-getfilestatus</name>
<value>true</value>
</property>
<property>
<name>fs.contract.supports-seek</name>
<value>true</value>