big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-18330. S3AFileSystem removes Path when calling createS3Client (#4572)

Browse Source 
Adds a new parameter object in s3ClientCreationParameters that holds 
the full s3a path URI

Contributed by Ashutosh Pant
This commit is contained in:
ashutoshpant 2022-07-21 05:16:39 -04:00 committed by GitHub
parent d07256a96d
commit bac2219e3c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 30 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
View File

@ -888,6 +888,7 @@ private void bindAWSClient(URI name, boolean dtEnabled) throws IOException {
S3ClientFactory.S3ClientCreationParameters parameters = null;
parameters = new S3ClientFactory.S3ClientCreationParameters()
.withCredentialSet(credentials)
.withPathUri(name)
.withEndpoint(endpoint)
.withMetrics(statisticsContext.newStatisticsFromAwsSdk())
.withPathStyleAccess(conf.getBoolean(PATH_STYLE_ACCESS, false))

27
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
View File

@ -115,6 +115,12 @@ final class S3ClientCreationParameters {
*/
private String userAgentSuffix = "";
/**
* S3A path.
* added in HADOOP-18330
*/
private URI pathUri;
/**
* List of request handlers to include in the chain
* of request execution in the SDK.
@ -264,5 +270,26 @@ public S3ClientCreationParameters withHeader(
public Map<String, String> getHeaders() {
return headers;
}
/**
* Get the full s3 path.
* added in HADOOP-18330
* @return path URI
*/
public URI getPathUri() {
return pathUri;
}
/**
* Set full s3a path.
* added in HADOOP-18330
* @param value new value
* @return the builder
*/
public S3ClientCreationParameters withPathUri(
final URI value) {
pathUri = value;
return this;
}
}
}

1
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java
View File

@ -176,6 +176,7 @@ private AmazonS3 createS3Client(Configuration conf,
S3ClientFactory.S3ClientCreationParameters parameters
= new S3ClientFactory.S3ClientCreationParameters()
.withCredentialSet(new AnonymousAWSCredentialsProvider())
.withPathUri(new URI("s3a://localhost/"))
.withEndpoint(endpoint)
.withMetrics(new EmptyS3AStatisticsContext()
.newStatisticsFromAwsSdk());

1
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java
View File

@ -589,6 +589,7 @@ protected ObjectMetadata readLandsatMetadata(final S3AFileSystem delegatedFS)
S3ClientFactory.S3ClientCreationParameters parameters = null;
parameters = new S3ClientFactory.S3ClientCreationParameters()
.withCredentialSet(testingCreds)
.withPathUri(new URI("s3a://localhost/"))
.withEndpoint(DEFAULT_ENDPOINT)
.withMetrics(new EmptyS3AStatisticsContext()
.newStatisticsFromAwsSdk())