From a8a58944bd7fa372ffde8ffcaf4f603762b1669f Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Tue, 30 Apr 2024 03:16:36 -0800 Subject: [PATCH] HADOOP-19146. S3A: noaa-cors-pds test bucket access with global endpoint fails (#6723) HADOOP-19057 switched the hadoop-aws test bucket from landsat-pds to noaa-cors-pds This new bucket isn't accessible if the client configuration sets an fs.s3a.endpoint/region value other than us-east-1. Contributed by Viraj Jasani --- .../s3a/ITestS3AAWSCredentialsProvider.java | 6 ++++ .../fs/s3a/ITestS3AFailureHandling.java | 10 +++---- .../fs/s3a/ITestS3APrefetchingCacheFiles.java | 8 ++++- .../auth/delegation/ITestDelegatedMRJob.java | 7 +++++ .../s3guard/AbstractS3GuardToolTestBase.java | 6 ++-- .../fs/s3a/s3guard/ITestS3GuardTool.java | 30 +++++++++++++++---- .../scale/ITestS3AInputStreamPerformance.java | 9 +++++- .../fs/s3a/test/PublicDatasetTestUtils.java | 12 ++++++++ 8 files changed, 72 insertions(+), 16 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java index 87a60c8c38..e6f258e556 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java @@ -44,6 +44,7 @@ import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.DELEGATION_TOKEN_BINDING; import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.CONSTRUCTOR_EXCEPTION; import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.getExternalData; +import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.isUsingDefaultExternalDataFile; import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.junit.Assert.*; @@ -207,6 +208,11 @@ public void testBadCredentialsWithRemap() throws Exception { @Test public void testAnonymousProvider() throws Exception { Configuration conf = createConf(AnonymousAWSCredentialsProvider.class); + if (isUsingDefaultExternalDataFile(conf)) { + removeBaseAndBucketOverrides(conf, + ENDPOINT); + conf.set(ENDPOINT, CENTRAL_ENDPOINT); + } Path testFile = getExternalData(conf); try (FileSystem fs = FileSystem.newInstance(testFile.toUri(), conf)) { Assertions.assertThat(fs) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java index 5f90115b8e..b550fc5864 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java @@ -32,9 +32,6 @@ import org.apache.hadoop.fs.store.audit.AuditSpan; import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.util.ArrayList; @@ -45,7 +42,9 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.*; import static org.apache.hadoop.fs.s3a.S3ATestUtils.createFiles; import static org.apache.hadoop.fs.s3a.S3ATestUtils.isBulkDeleteEnabled; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; import static org.apache.hadoop.fs.s3a.test.ExtraAssertions.failIf; +import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.isUsingDefaultExternalDataFile; import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.requireDefaultExternalData; import static org.apache.hadoop.test.LambdaTestUtils.*; import static org.apache.hadoop.util.functional.RemoteIterators.mappingRemoteIterator; @@ -55,14 +54,15 @@ * ITest for failure handling, primarily multipart deletion. */ public class ITestS3AFailureHandling extends AbstractS3ATestBase { - private static final Logger LOG = - LoggerFactory.getLogger(ITestS3AFailureHandling.class); @Override protected Configuration createConfiguration() { Configuration conf = super.createConfiguration(); S3ATestUtils.disableFilesystemCaching(conf); conf.setBoolean(Constants.ENABLE_MULTI_DELETE, true); + if (isUsingDefaultExternalDataFile(conf)) { + removeBaseAndBucketOverrides(conf, Constants.ENDPOINT); + } return conf; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingCacheFiles.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingCacheFiles.java index ce962483d5..5e6731ed52 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingCacheFiles.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingCacheFiles.java @@ -38,9 +38,11 @@ import org.apache.hadoop.fs.s3a.performance.AbstractS3ACostTest; import static org.apache.hadoop.fs.s3a.Constants.BUFFER_DIR; +import static org.apache.hadoop.fs.s3a.Constants.ENDPOINT; import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_BLOCK_SIZE_KEY; import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_ENABLED_KEY; import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.getExternalData; +import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.isUsingDefaultExternalDataFile; import static org.apache.hadoop.io.IOUtils.cleanupWithLogger; /** @@ -84,7 +86,11 @@ public void setUp() throws Exception { @Override public Configuration createConfiguration() { Configuration configuration = super.createConfiguration(); - S3ATestUtils.removeBaseAndBucketOverrides(configuration, PREFETCH_ENABLED_KEY); + if (isUsingDefaultExternalDataFile(configuration)) { + S3ATestUtils.removeBaseAndBucketOverrides(configuration, + PREFETCH_ENABLED_KEY, + ENDPOINT); + } configuration.setBoolean(PREFETCH_ENABLED_KEY, true); // use a small block size unless explicitly set in the test config. configuration.setInt(PREFETCH_BLOCK_SIZE_KEY, BLOCK_SIZE); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestDelegatedMRJob.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestDelegatedMRJob.java index ba9746358c..4aaf35f061 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestDelegatedMRJob.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestDelegatedMRJob.java @@ -33,6 +33,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.s3a.Constants; import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; @@ -53,12 +54,14 @@ import static org.apache.hadoop.fs.s3a.S3ATestUtils.deployService; import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching; import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestPropertyInt; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; import static org.apache.hadoop.fs.s3a.S3ATestUtils.terminateService; import static org.apache.hadoop.fs.s3a.auth.RoleTestUtils.probeForAssumedRoleARN; import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.*; import static org.apache.hadoop.fs.s3a.auth.delegation.MiniKerberizedHadoopCluster.assertSecurityEnabled; import static org.apache.hadoop.fs.s3a.auth.delegation.MiniKerberizedHadoopCluster.closeUserFileSystems; import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.getOrcData; +import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.isUsingDefaultExternalDataFile; import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.requireAnonymousDataPath; /** @@ -251,6 +254,10 @@ public void testCommonCrawlLookup() throws Throwable { public void testJobSubmissionCollectsTokens() throws Exception { describe("Mock Job test"); JobConf conf = new JobConf(getConfiguration()); + if (isUsingDefaultExternalDataFile(conf)) { + removeBaseAndBucketOverrides(conf, + Constants.ENDPOINT); + } // the input here is the external file; which lets // us differentiate source URI from dest URI diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java index fd7a528a5d..22a4727739 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java @@ -118,12 +118,12 @@ protected int run(Object... args) throws Exception { * Run a S3GuardTool command from a varags list, catch any raised * ExitException and verify the status code matches that expected. * @param status expected status code of the exception + * @param conf configuration object. * @param args argument list * @throws Exception any exception */ - protected void runToFailure(int status, Object... args) + protected void runToFailure(int status, Configuration conf, Object... args) throws Exception { - final Configuration conf = getConfiguration(); ExitUtil.ExitException ex = intercept(ExitUtil.ExitException.class, () -> runS3GuardCommand(conf, args)); @@ -247,7 +247,7 @@ public void testUnsupported() throws Throwable { describe("Verify the unsupported tools are rejected"); for (String tool : UNSUPPORTED_COMMANDS) { describe("Probing %s", tool); - runToFailure(E_S3GUARD_UNSUPPORTED, tool); + runToFailure(E_S3GUARD_UNSUPPORTED, getConfiguration(), tool); } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardTool.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardTool.java index 18fdccabae..59787617b8 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardTool.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardTool.java @@ -37,6 +37,7 @@ import org.apache.hadoop.util.StringUtils; import static org.apache.hadoop.fs.contract.ContractTestUtils.skip; +import static org.apache.hadoop.fs.s3a.Constants.ENDPOINT; import static org.apache.hadoop.fs.s3a.Constants.FIPS_ENDPOINT; import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_ALGORITHM; import static org.apache.hadoop.fs.s3a.MultipartTestUtils.assertNoUploadsAt; @@ -48,6 +49,7 @@ import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.E_BAD_STATE; import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.Uploads; import static org.apache.hadoop.fs.s3a.s3guard.S3GuardToolTestHelper.exec; +import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.isUsingDefaultExternalDataFile; /** * Test S3Guard Tool CLI commands. @@ -60,8 +62,13 @@ public class ITestS3GuardTool extends AbstractS3GuardToolTestBase { @Test public void testExternalBucketRequireUnencrypted() throws Throwable { - removeBaseAndBucketOverrides(getConfiguration(), S3_ENCRYPTION_ALGORITHM); - run(BucketInfo.NAME, + Configuration conf = getConfiguration(); + if (isUsingDefaultExternalDataFile(conf)) { + removeBaseAndBucketOverrides(conf, + S3_ENCRYPTION_ALGORITHM, + ENDPOINT); + } + run(conf, BucketInfo.NAME, "-" + BucketInfo.ENCRYPTION_FLAG, "none", externalBucket()); } @@ -81,10 +88,17 @@ private String externalBucket() { @Test public void testExternalBucketRequireEncrypted() throws Throwable { + Configuration conf = getConfiguration(); + if (isUsingDefaultExternalDataFile(conf)) { + removeBaseAndBucketOverrides(conf, + ENDPOINT); + } runToFailure(E_BAD_STATE, + conf, BucketInfo.NAME, "-" + BucketInfo.ENCRYPTION_FLAG, - "AES256", externalBucket()); + "AES256", + externalBucket()); } @Test @@ -212,9 +226,13 @@ public void testUploadListByAge() throws Throwable { @Test public void testUploadNegativeExpect() throws Throwable { - runToFailure(E_BAD_STATE, Uploads.NAME, "-expect", "1", - path("/we/are/almost/postive/this/doesnt/exist/fhfsadfoijew") - .toString()); + Configuration conf = getConfiguration(); + runToFailure(E_BAD_STATE, + conf, + Uploads.NAME, + "-expect", + "1", + path("/we/are/almost/postive/this/doesnt/exist/fhfsadfoijew").toString()); } private void assertNumUploads(Path path, int numUploads) throws Exception { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java index ae09452372..a787f52bd4 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java @@ -60,6 +60,7 @@ import static org.apache.hadoop.fs.s3a.S3ATestUtils.assume; import static org.apache.hadoop.fs.s3a.S3ATestUtils.getInputStreamStatistics; import static org.apache.hadoop.fs.s3a.S3ATestUtils.getS3AInputStream; +import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.isUsingDefaultExternalDataFile; import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.assertThatStatisticMinimum; import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.lookupMaximumStatistic; import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.lookupMeanStatistic; @@ -99,7 +100,13 @@ public class ITestS3AInputStreamPerformance extends S3AScaleTestBase { @Override protected Configuration createScaleConfiguration() { Configuration conf = super.createScaleConfiguration(); - S3ATestUtils.removeBaseAndBucketOverrides(conf, PREFETCH_ENABLED_KEY); + S3ATestUtils.removeBaseAndBucketOverrides(conf, + PREFETCH_ENABLED_KEY); + if (isUsingDefaultExternalDataFile(conf)) { + S3ATestUtils.removeBaseAndBucketOverrides( + conf, + ENDPOINT); + } conf.setBoolean(PREFETCH_ENABLED_KEY, false); return conf; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/PublicDatasetTestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/PublicDatasetTestUtils.java index 3835548b1e..9400cfb3fb 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/PublicDatasetTestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/PublicDatasetTestUtils.java @@ -133,6 +133,18 @@ public static String requireDefaultExternalDataFile(Configuration conf) { return filename; } + /** + * To determine whether {@value S3ATestConstants#KEY_CSVTEST_FILE} is configured to be + * different from the default external file. + * + * @param conf Configuration object. + * @return True if the default external data file is being used. + */ + public static boolean isUsingDefaultExternalDataFile(final Configuration conf) { + final String filename = getExternalData(conf).toUri().toString(); + return DEFAULT_EXTERNAL_FILE.equals(filename); + } + /** * Get the test external file; assume() that it is not modified (i.e. we haven't * switched to a new storage infrastructure where the bucket is no longer