big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-18068. upgrade AWS SDK to 1.12.132 (#3864)

Browse Source 
With this update, the versions of key shaded dependencies are

  jackson    2.12.3
  httpclient 4.5.13

Contributed by Steve Loughran
This commit is contained in:
Steve Loughran 2022-01-18 10:31:28 +00:00 committed by GitHub
parent 14ba19af06
commit d8ab84275e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 18 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
NOTICE-binary
View File

@ -66,7 +66,7 @@ available from http://www.digip.org/jansson/.
AWS SDK for Java AWS SDK for Java
Copyright 2010-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. Copyright 2010-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
This product includes software developed by This product includes software developed by
Amazon Technologies, Inc (http://www.amazon.com/). Amazon Technologies, Inc (http://www.amazon.com/).

2
hadoop-project/pom.xml
View File

@ -185,7 +185,7 @@
<exec-maven-plugin.version>1.3.1</exec-maven-plugin.version> <exec-maven-plugin.version>1.3.1</exec-maven-plugin.version>
<make-maven-plugin.version>1.0-beta-1</make-maven-plugin.version> <make-maven-plugin.version>1.0-beta-1</make-maven-plugin.version>
<surefire.fork.timeout>900</surefire.fork.timeout> <surefire.fork.timeout>900</surefire.fork.timeout>
<aws-java-sdk.version>1.11.1026</aws-java-sdk.version> <aws-java-sdk.version>1.12.132</aws-java-sdk.version>
<hsqldb.version>2.3.4</hsqldb.version> <hsqldb.version>2.3.4</hsqldb.version>
<frontend-maven-plugin.version>1.11.2</frontend-maven-plugin.version> <frontend-maven-plugin.version>1.11.2</frontend-maven-plugin.version>
<jasmine-maven-plugin.version>2.1</jasmine-maven-plugin.version> <jasmine-maven-plugin.version>2.1</jasmine-maven-plugin.version>

1
hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
View File

@ -1249,6 +1249,7 @@ to AWS services.
* Try and get other people, especially anyone with their own endpoints, * Try and get other people, especially anyone with their own endpoints,
apps or different deployment environments, to run their own tests. apps or different deployment environments, to run their own tests.
* Run the load tests, especially `ILoadTestS3ABulkDeleteThrottling`. * Run the load tests, especially `ILoadTestS3ABulkDeleteThrottling`.
* Checkout cloudstore, build it against your version of hadoop, then use its CLI to run some commands (`storediag` etc)
### Dealing with Deprecated APIs and New Features ### Dealing with Deprecated APIs and New Features

19
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java
View File

@ -19,6 +19,7 @@
package org.apache.hadoop.fs.s3a; package org.apache.hadoop.fs.s3a;
import com.amazonaws.regions.Regions; import com.amazonaws.regions.Regions;
import org.assertj.core.api.Assertions;
import org.junit.Test; import org.junit.Test;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -27,6 +28,14 @@
import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.apache.hadoop.test.LambdaTestUtils.intercept;
/**
* Verifies the mapping of ARN declaration of resource to the associated
* access point.
* The region mapping assertions have been brittle to changes across AWS SDK
* versions to only verify partial matches, rather than the FQDN of the
* endpoints.
*
*/
public class TestArnResource extends HadoopTestBase { public class TestArnResource extends HadoopTestBase {
private final static Logger LOG = LoggerFactory.getLogger(TestArnResource.class); private final static Logger LOG = LoggerFactory.getLogger(TestArnResource.class);
@ -37,10 +46,10 @@ public void parseAccessPointFromArn() throws IllegalArgumentException {
String accessPoint = "testAp"; String accessPoint = "testAp";
String accountId = "123456789101"; String accountId = "123456789101";
String[][] regionPartitionEndpoints = new String[][] { String[][] regionPartitionEndpoints = new String[][] {
{Regions.EU_WEST_1.getName(), "aws", "s3-accesspoint.eu-west-1.amazonaws.com"}, {Regions.EU_WEST_1.getName(), "aws", "eu-west-1.amazonaws.com"},
{Regions.US_GOV_EAST_1.getName(), "aws-us-gov", {Regions.US_GOV_EAST_1.getName(), "aws-us-gov",
"s3-accesspoint.us-gov-east-1.amazonaws.com"}, "us-gov-east-1.amazonaws.com"},
{Regions.CN_NORTH_1.getName(), "aws-cn", "s3-accesspoint.cn-north-1.amazonaws.com.cn"}, {Regions.CN_NORTH_1.getName(), "aws-cn", "cn-north-1.amazonaws.com"},
}; };
for (String[] testPair : regionPartitionEndpoints) { for (String[] testPair : regionPartitionEndpoints) {
@ -57,7 +66,9 @@ public void parseAccessPointFromArn() throws IllegalArgumentException {
assertEquals("Access Point name does not match", accessPoint, resource.getName()); assertEquals("Access Point name does not match", accessPoint, resource.getName());
assertEquals("Account Id does not match", accountId, resource.getOwnerAccountId()); assertEquals("Account Id does not match", accountId, resource.getOwnerAccountId());
assertEquals("Region does not match", region, resource.getRegion()); assertEquals("Region does not match", region, resource.getRegion());
assertEquals("Endpoint does not match", endpoint, resource.getEndpoint()); Assertions.assertThat(resource.getEndpoint())
.describedAs("Endpoint does not match")
.contains(endpoint);
} }
} }