big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-18328. S3A to support S3 on Outposts (#4533)

Browse Source 
Contributed by Sotetsu Suzugamine
This commit is contained in:
suzu 2023-08-23 19:38:07 +09:00 committed by GitHub
parent 43c889636a
commit 70b6c155bc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 27 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java
View File

@ -26,7 +26,8 @@
* Represents an Arn Resource, this can be an accesspoint or bucket.
*/
public final class ArnResource {
private final static String ACCESSPOINT_ENDPOINT_FORMAT = "s3-accesspoint.%s.amazonaws.com";
private final static String S3_ACCESSPOINT_ENDPOINT_FORMAT = "s3-accesspoint.%s.amazonaws.com";
private final static String S3_OUTPOSTS_ACCESSPOINT_ENDPOINT_FORMAT = "s3-outposts.%s.amazonaws.com";
/**
* Resource name.
@ -69,6 +70,10 @@ private ArnResource(String name, String owner, String region, String partition,
this.accessPointRegionKey = String.format("accesspoint-%s", region);
}
private boolean isOutposts(){
return fullArn.contains("s3-outposts");
}
/**
* Resource name.
* @return resource name.
@ -106,7 +111,8 @@ public String getFullArn() {
* @return resource endpoint.
*/
public String getEndpoint() {
return String.format(ACCESSPOINT_ENDPOINT_FORMAT, region);
String format = isOutposts() ? S3_OUTPOSTS_ACCESSPOINT_ENDPOINT_FORMAT : S3_ACCESSPOINT_ENDPOINT_FORMAT;
return String.format(format, region);
}
/**

24
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java
View File

@ -56,7 +56,7 @@ public void parseAccessPointFromArn() throws IllegalArgumentException {
String region = testPair[0];
String partition = testPair[1];
ArnResource resource = getArnResourceFrom(partition, region, MOCK_ACCOUNT, accessPoint);
ArnResource resource = getArnResourceFrom(partition, "s3", region, MOCK_ACCOUNT, accessPoint);
assertEquals("Access Point name does not match", accessPoint, resource.getName());
assertEquals("Account Id does not match", MOCK_ACCOUNT, resource.getOwnerAccountId());
assertEquals("Region does not match", region, resource.getRegion());
@ -64,10 +64,10 @@ public void parseAccessPointFromArn() throws IllegalArgumentException {
}
@Test
public void makeSureEndpointHasTheCorrectFormat() {
public void makeSureS3EndpointHasTheCorrectFormat() {
// Access point (AP) endpoints are different from S3 bucket endpoints, thus when using APs the
// endpoints for the client are modified. This test makes sure endpoint is set up correctly.
ArnResource accessPoint = getArnResourceFrom("aws", "eu-west-1", MOCK_ACCOUNT,
ArnResource accessPoint = getArnResourceFrom("aws", "s3", "eu-west-1", MOCK_ACCOUNT,
"test");
String expected = "s3-accesspoint.eu-west-1.amazonaws.com";
@ -76,6 +76,19 @@ public void makeSureEndpointHasTheCorrectFormat() {
.isEqualTo(expected);
}
@Test
public void makeSureS3OutpostsEndpointHasTheCorrectFormat() {
// Access point (AP) endpoints are different from S3 bucket endpoints, thus when using APs the
// endpoints for the client are modified. This test makes sure endpoint is set up correctly.
ArnResource accessPoint = getArnResourceFrom("aws", "s3-outposts", "eu-west-1", MOCK_ACCOUNT,
"test");
String expected = "s3-outposts.eu-west-1.amazonaws.com";
Assertions.assertThat(accessPoint.getEndpoint())
.describedAs("Endpoint has invalid format. Access Point requests will not work")
.isEqualTo(expected);
}
@Test
public void invalidARNsMustThrow() throws Exception {
describe("Using an invalid ARN format must throw when initializing an ArnResource.");
@ -87,15 +100,16 @@ public void invalidARNsMustThrow() throws Exception {
/**
* Create an {@link ArnResource} from string components
* @param partition - partition for ARN
* @param service - service for ARN
* @param region - region for ARN
* @param accountId - accountId for ARN
* @param resourceName - ARN resource name
* @return ArnResource described by its properties
*/
private ArnResource getArnResourceFrom(String partition, String region, String accountId,
private ArnResource getArnResourceFrom(String partition, String service, String region, String accountId,
String resourceName) {
// arn:partition:service:region:account-id:resource-type/resource-id
String arn = String.format("arn:%s:s3:%s:%s:accesspoint/%s", partition, region, accountId,
String arn = String.format("arn:%s:%s:%s:%s:accesspoint/%s", partition, service, region, accountId,
resourceName);
return ArnResource.accessPointFromArn(arn);