big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-18696. ITestS3ABucketExistence arn test failures. (#5557)

Browse Source 
Explicitly sets the fs.s3a.endpoint.region to eu-west-1 so
the ARN-referenced fs creation fails with unknown store
rather than IllegalArgumentException.

Steve Loughran
This commit is contained in:
Steve Loughran 2023-04-17 10:18:33 +01:00 committed by GitHub
parent 2c4d6bf33d
commit 6ea10cf41b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 15 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABucketExistence.java
View File

@ -34,6 +34,7 @@
import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset; import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset;
import static org.apache.hadoop.fs.s3a.Constants.AWS_REGION;
import static org.apache.hadoop.fs.s3a.Constants.AWS_S3_ACCESSPOINT_REQUIRED; import static org.apache.hadoop.fs.s3a.Constants.AWS_S3_ACCESSPOINT_REQUIRED;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A; import static org.apache.hadoop.fs.s3a.Constants.FS_S3A;
import static org.apache.hadoop.fs.s3a.Constants.S3A_BUCKET_PROBE; import static org.apache.hadoop.fs.s3a.Constants.S3A_BUCKET_PROBE;
@ -47,7 +48,7 @@ public class ITestS3ABucketExistence extends AbstractS3ATestBase {
private FileSystem fs; private FileSystem fs;
private final String randomBucket = private final String randomBucket =
"random-bucket-" + UUID.randomUUID().toString(); "random-bucket-" + UUID.randomUUID();
private final URI uri = URI.create(FS_S3A + "://" + randomBucket + "/"); private final URI uri = URI.create(FS_S3A + "://" + randomBucket + "/");
@ -163,7 +164,7 @@ public void testBucketProbingParameterValidation() throws Exception {
@Test @Test
public void testAccessPointProbingV2() throws Exception { public void testAccessPointProbingV2() throws Exception {
describe("Test V2 bucket probing using an AccessPoint ARN"); describe("Test V2 bucket probing using an AccessPoint ARN");
Configuration configuration = createConfigurationWithProbe(2); Configuration configuration = createArnConfiguration();
String accessPointArn = "arn:aws:s3:eu-west-1:123456789012:accesspoint/" + randomBucket; String accessPointArn = "arn:aws:s3:eu-west-1:123456789012:accesspoint/" + randomBucket;
configuration.set(String.format(InternalConstants.ARN_BUCKET_OPTION, randomBucket), configuration.set(String.format(InternalConstants.ARN_BUCKET_OPTION, randomBucket),
accessPointArn); accessPointArn);
@ -175,7 +176,7 @@ public void testAccessPointProbingV2() throws Exception {
@Test @Test
public void testAccessPointRequired() throws Exception { public void testAccessPointRequired() throws Exception {
describe("Test V2 bucket probing with 'fs.s3a.accesspoint.required' property."); describe("Test V2 bucket probing with 'fs.s3a.accesspoint.required' property.");
Configuration configuration = createConfigurationWithProbe(2); Configuration configuration = createArnConfiguration();
configuration.set(AWS_S3_ACCESSPOINT_REQUIRED, "true"); configuration.set(AWS_S3_ACCESSPOINT_REQUIRED, "true");
intercept(PathIOException.class, intercept(PathIOException.class,
InternalConstants.AP_REQUIRED_EXCEPTION, InternalConstants.AP_REQUIRED_EXCEPTION,
@ -189,6 +190,17 @@ public void testAccessPointRequired() throws Exception {
() -> FileSystem.get(uri, configuration)); () -> FileSystem.get(uri, configuration));
} }
/**
* Create a configuration which has bucket probe 2 and the endpoint.region
* option set to "eu-west-1" to match that of the ARNs generated.
* @return a configuration for tests which are expected to fail in specific ways.
*/
private Configuration createArnConfiguration() {
Configuration configuration = createConfigurationWithProbe(2);
configuration.set(AWS_REGION, "eu-west-1");
return configuration;
}
@Override @Override
protected Configuration getConfiguration() { protected Configuration getConfiguration() {
Configuration configuration = super.getConfiguration(); Configuration configuration = super.getConfiguration();