big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-18085. S3 SDK Upgrade causes AccessPoint ARN endpoint mistranslation (#3902)

Browse Source 
Part of HADOOP-17198. Support S3 Access Points.

HADOOP-18068. "upgrade AWS SDK to 1.12.132" broke the access point endpoint
translation.

Correct endpoints should start with "s3-accesspoint.", after SDK upgrade they start with
"s3.accesspoint-" which messes up tests + region detection by the SDK.

Contributed by Bogdan Stolojan
This commit is contained in:
Petre Bogdan Stolojan 2022-02-04 15:37:08 +00:00 committed by GitHub
parent 3684c7f66a
commit 5e7ce26e66
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 39 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java
View File

@ -21,12 +21,12 @@
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import com.amazonaws.arn.Arn; import com.amazonaws.arn.Arn;
import com.amazonaws.regions.RegionUtils;
/** /**
* Represents an Arn Resource, this can be an accesspoint or bucket. * Represents an Arn Resource, this can be an accesspoint or bucket.
*/ */
public final class ArnResource { public final class ArnResource {
private final static String ACCESSPOINT_ENDPOINT_FORMAT = "s3-accesspoint.%s.amazonaws.com";
/** /**
* Resource name. * Resource name.
@ -106,8 +106,7 @@ public String getFullArn() {
* @return resource endpoint. * @return resource endpoint.
*/ */
public String getEndpoint() { public String getEndpoint() {
return RegionUtils.getRegion(accessPointRegionKey) return String.format(ACCESSPOINT_ENDPOINT_FORMAT, region);
.getServiceEndpoint("s3");
} }
/** /**

53
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java
View File

@ -39,39 +39,43 @@
public class TestArnResource extends HadoopTestBase { public class TestArnResource extends HadoopTestBase {
private final static Logger LOG = LoggerFactory.getLogger(TestArnResource.class); private final static Logger LOG = LoggerFactory.getLogger(TestArnResource.class);
private final static String MOCK_ACCOUNT = "123456789101";
@Test @Test
public void parseAccessPointFromArn() throws IllegalArgumentException { public void parseAccessPointFromArn() throws IllegalArgumentException {
describe("Parse AccessPoint ArnResource from arn string"); describe("Parse AccessPoint ArnResource from arn string");
String accessPoint = "testAp"; String accessPoint = "testAp";
String accountId = "123456789101";
String[][] regionPartitionEndpoints = new String[][] { String[][] regionPartitionEndpoints = new String[][] {
{Regions.EU_WEST_1.getName(), "aws", "eu-west-1.amazonaws.com"}, {Regions.EU_WEST_1.getName(), "aws"},
{Regions.US_GOV_EAST_1.getName(), "aws-us-gov", {Regions.US_GOV_EAST_1.getName(), "aws-us-gov"},
"us-gov-east-1.amazonaws.com"}, {Regions.CN_NORTH_1.getName(), "aws-cn"},
{Regions.CN_NORTH_1.getName(), "aws-cn", "cn-north-1.amazonaws.com"},
}; };
for (String[] testPair : regionPartitionEndpoints) { for (String[] testPair : regionPartitionEndpoints) {
String region = testPair[0]; String region = testPair[0];
String partition = testPair[1]; String partition = testPair[1];
String endpoint = testPair[2];
// arn:partition:service:region:account-id:resource-type/resource-id ArnResource resource = getArnResourceFrom(partition, region, MOCK_ACCOUNT, accessPoint);
String arn = String.format("arn:%s:s3:%s:%s:accesspoint/%s", partition, region, accountId,
accessPoint);
ArnResource resource = ArnResource.accessPointFromArn(arn);
assertEquals("Arn does not match", arn, resource.getFullArn());
assertEquals("Access Point name does not match", accessPoint, resource.getName()); assertEquals("Access Point name does not match", accessPoint, resource.getName());
assertEquals("Account Id does not match", accountId, resource.getOwnerAccountId()); assertEquals("Account Id does not match", MOCK_ACCOUNT, resource.getOwnerAccountId());
assertEquals("Region does not match", region, resource.getRegion()); assertEquals("Region does not match", region, resource.getRegion());
Assertions.assertThat(resource.getEndpoint())
.describedAs("Endpoint does not match")
.contains(endpoint);
} }
} }
@Test
public void makeSureEndpointHasTheCorrectFormat() {
// Access point (AP) endpoints are different from S3 bucket endpoints, thus when using APs the
// endpoints for the client are modified. This test makes sure endpoint is set up correctly.
ArnResource accessPoint = getArnResourceFrom("aws", "eu-west-1", MOCK_ACCOUNT,
"test");
String expected = "s3-accesspoint.eu-west-1.amazonaws.com";
Assertions.assertThat(accessPoint.getEndpoint())
.describedAs("Endpoint has invalid format. Access Point requests will not work")
.isEqualTo(expected);
}
@Test @Test
public void invalidARNsMustThrow() throws Exception { public void invalidARNsMustThrow() throws Exception {
describe("Using an invalid ARN format must throw when initializing an ArnResource."); describe("Using an invalid ARN format must throw when initializing an ArnResource.");
@ -80,6 +84,23 @@ public void invalidARNsMustThrow() throws Exception {
ArnResource.accessPointFromArn("invalid:arn:resource")); ArnResource.accessPointFromArn("invalid:arn:resource"));
} }
/**
* Create an {@link ArnResource} from string components
* @param partition - partition for ARN
* @param region - region for ARN
* @param accountId - accountId for ARN
* @param resourceName - ARN resource name
* @return ArnResource described by its properties
*/
private ArnResource getArnResourceFrom(String partition, String region, String accountId,
String resourceName) {
// arn:partition:service:region:account-id:resource-type/resource-id
String arn = String.format("arn:%s:s3:%s:%s:accesspoint/%s", partition, region, accountId,
resourceName);
return ArnResource.accessPointFromArn(arn);
}
private void describe(String message) { private void describe(String message) {
LOG.info(message); LOG.info(message);
} }