big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-19286: S3A: Support cross region access when S3 region/endpoint is set (ADDENDUM) (#7098)

Browse Source 
Contributed by Syed Shameerur Rahman
This commit is contained in:
Syed Shameerur Rahman 2024-10-04 19:28:53 +05:30 committed by GitHub
parent 4e6432a0ab
commit 5ea3a1bd0a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 8 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
View File

@ -271,8 +271,10 @@ protected ClientOverrideConfiguration.Builder createClientOverrideConfiguration(
* <li>If endpoint is configured via via fs.s3a.endpoint, set it. * <li>If endpoint is configured via via fs.s3a.endpoint, set it.
* If no region is configured, try to parse region from endpoint. </li> * If no region is configured, try to parse region from endpoint. </li>
* <li> If no region is configured, and it could not be parsed from the endpoint, * <li> If no region is configured, and it could not be parsed from the endpoint,
* set the default region as US_EAST_2 and enable cross region access. </li> * set the default region as US_EAST_2</li>
* <li> If configured region is empty, fallback to SDK resolution chain. </li> * <li> If configured region is empty, fallback to SDK resolution chain. </li>
* <li> S3 cross region is enabled by default irrespective of region or endpoint
* is set or not.</li>
* </ol> * </ol>
* *
* @param builder S3 client builder. * @param builder S3 client builder.

5
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java
View File

@ -356,6 +356,10 @@ public void testWithOutCrossRegionAccess() throws Exception {
// skip the test if the region is sa-east-1 // skip the test if the region is sa-east-1
skipCrossRegionTest(); skipCrossRegionTest();
final Configuration newConf = new Configuration(getConfiguration()); final Configuration newConf = new Configuration(getConfiguration());
removeBaseAndBucketOverrides(newConf,
ENDPOINT,
AWS_S3_CROSS_REGION_ACCESS_ENABLED,
AWS_REGION);
// disable cross region access // disable cross region access
newConf.setBoolean(AWS_S3_CROSS_REGION_ACCESS_ENABLED, false); newConf.setBoolean(AWS_S3_CROSS_REGION_ACCESS_ENABLED, false);
newConf.set(AWS_REGION, SA_EAST_1); newConf.set(AWS_REGION, SA_EAST_1);
@ -374,6 +378,7 @@ public void testWithCrossRegionAccess() throws Exception {
skipCrossRegionTest(); skipCrossRegionTest();
final Configuration newConf = new Configuration(getConfiguration()); final Configuration newConf = new Configuration(getConfiguration());
removeBaseAndBucketOverrides(newConf, removeBaseAndBucketOverrides(newConf,
ENDPOINT,
AWS_S3_CROSS_REGION_ACCESS_ENABLED, AWS_S3_CROSS_REGION_ACCESS_ENABLED,
AWS_REGION); AWS_REGION);
// enable cross region access // enable cross region access