big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-17105. S3AFS - Do not attempt to resolve symlinks in globStatus (#2113)

Browse Source 
Contributed by Jimmy Zuber.
This commit is contained in:
jimmy-zuber-amzn 2020-07-13 11:07:48 -07:00 committed by GitHub
parent b9fa5e0182
commit 806d84b79c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 50 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
View File

@ -3980,6 +3980,8 @@ public boolean isMagicCommitPath(Path path) {
/**
* Increments the statistic {@link Statistic#INVOCATION_GLOB_STATUS}.
* Override superclass so as to disable symlink resolution as symlinks
* are not supported by S3A.
* {@inheritDoc}
*/
@Override
@ -3988,9 +3990,9 @@ public FileStatus[] globStatus(Path pathPattern) throws IOException {
}
/**
* Override superclass so as to disable symlink resolution and so avoid
* some calls to the FS which may have problems when the store is being
* inconsistent.
* Increments the statistic {@link Statistic#INVOCATION_GLOB_STATUS}.
* Override superclass so as to disable symlink resolution as symlinks
* are not supported by S3A.
* {@inheritDoc}
*/
@Override
@ -4002,7 +4004,7 @@ public FileStatus[] globStatus(
return Globber.createGlobber(this)
.withPathPattern(pathPattern)
.withPathFiltern(filter)
.withResolveSymlinks(true)
.withResolveSymlinks(false)
.build()
.glob();
}

44
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java
View File

@ -574,4 +574,48 @@ public void testCreateCost() throws Throwable {
}
}
@Test
public void testCostOfGlobStatus() throws Throwable {
describe("Test globStatus has expected cost");
S3AFileSystem fs = getFileSystem();
assume("Unguarded FS only", !fs.hasMetadataStore());
Path basePath = path("testCostOfGlobStatus/nextFolder/");
// create a bunch of files
int filesToCreate = 10;
for (int i = 0; i < filesToCreate; i++) {
try (FSDataOutputStream out = fs.create(basePath.suffix("/" + i))) {
verifyOperationCount(1, 1);
}
}
fs.globStatus(basePath.suffix("/*"));
// 2 head + 1 list from getFileStatus on path,
// plus 1 list to match the glob pattern
verifyOperationCount(2, 2);
}
@Test
public void testCostOfGlobStatusNoSymlinkResolution() throws Throwable {
describe("Test globStatus does not attempt to resolve symlinks");
S3AFileSystem fs = getFileSystem();
assume("Unguarded FS only", !fs.hasMetadataStore());
Path basePath = path("testCostOfGlobStatusNoSymlinkResolution/f/");
// create a single file, globStatus returning a single file on a pattern
// triggers attempts at symlinks resolution if configured
String fileName = "/notASymlinkDOntResolveMeLikeOne";
try (FSDataOutputStream out = fs.create(basePath.suffix(fileName))) {
verifyOperationCount(1, 1);
}
fs.globStatus(basePath.suffix("/*"));
// unguarded: 2 head + 1 list from getFileStatus on path,
// plus 1 list to match the glob pattern
// no additional operations from symlink resolution
verifyOperationCount(2, 2);
}
}