HADOOP-19146. S3A: noaa-cors-pds test bucket access with global endpoint fails (#6723)

HADOOP-19057 switched the hadoop-aws test bucket from landsat-pds to 
noaa-cors-pds 

This new bucket isn't accessible if the client configuration
sets an fs.s3a.endpoint/region value other than us-east-1.

Contributed by Viraj Jasani

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java

@@ -44,6 +44,7 @@
 import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.DELEGATION_TOKEN_BINDING;
 import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.CONSTRUCTOR_EXCEPTION;
 import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.getExternalData;
+import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.isUsingDefaultExternalDataFile;
 import static org.apache.hadoop.test.LambdaTestUtils.intercept;
 import static org.junit.Assert.*;
 
@@ -207,6 +208,11 @@ public void testBadCredentialsWithRemap() throws Exception {
   @Test
   public void testAnonymousProvider() throws Exception {
     Configuration conf = createConf(AnonymousAWSCredentialsProvider.class);
+    if (isUsingDefaultExternalDataFile(conf)) {
+      removeBaseAndBucketOverrides(conf,
+          ENDPOINT);
+      conf.set(ENDPOINT, CENTRAL_ENDPOINT);
+    }
     Path testFile = getExternalData(conf);
     try (FileSystem fs = FileSystem.newInstance(testFile.toUri(), conf)) {
       Assertions.assertThat(fs)

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java

@@ -32,9 +32,6 @@
 import org.apache.hadoop.fs.store.audit.AuditSpan;
 
 import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -45,7 +42,9 @@
 import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.createFiles;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.isBulkDeleteEnabled;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
 import static org.apache.hadoop.fs.s3a.test.ExtraAssertions.failIf;
+import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.isUsingDefaultExternalDataFile;
 import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.requireDefaultExternalData;
 import static org.apache.hadoop.test.LambdaTestUtils.*;
 import static org.apache.hadoop.util.functional.RemoteIterators.mappingRemoteIterator;
@@ -55,14 +54,15 @@
  * ITest for failure handling, primarily multipart deletion.
  */
 public class ITestS3AFailureHandling extends AbstractS3ATestBase {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(ITestS3AFailureHandling.class);
 
   @Override
   protected Configuration createConfiguration() {
     Configuration conf = super.createConfiguration();
     S3ATestUtils.disableFilesystemCaching(conf);
     conf.setBoolean(Constants.ENABLE_MULTI_DELETE, true);
+    if (isUsingDefaultExternalDataFile(conf)) {
+      removeBaseAndBucketOverrides(conf, Constants.ENDPOINT);
+    }
     return conf;
   }
 

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingCacheFiles.java

@@ -38,9 +38,11 @@
 import org.apache.hadoop.fs.s3a.performance.AbstractS3ACostTest;
 
 import static org.apache.hadoop.fs.s3a.Constants.BUFFER_DIR;
+import static org.apache.hadoop.fs.s3a.Constants.ENDPOINT;
 import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_BLOCK_SIZE_KEY;
 import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_ENABLED_KEY;
 import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.getExternalData;
+import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.isUsingDefaultExternalDataFile;
 import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
 
 /**
@@ -84,7 +86,11 @@ public void setUp() throws Exception {
   @Override
   public Configuration createConfiguration() {
     Configuration configuration = super.createConfiguration();
-    S3ATestUtils.removeBaseAndBucketOverrides(configuration, PREFETCH_ENABLED_KEY);
+    if (isUsingDefaultExternalDataFile(configuration)) {
+      S3ATestUtils.removeBaseAndBucketOverrides(configuration,
+          PREFETCH_ENABLED_KEY,
+          ENDPOINT);
+    }
     configuration.setBoolean(PREFETCH_ENABLED_KEY, true);
     // use a small block size unless explicitly set in the test config.
     configuration.setInt(PREFETCH_BLOCK_SIZE_KEY, BLOCK_SIZE);

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestDelegatedMRJob.java

@@ -33,6 +33,7 @@
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.Constants;
 import org.apache.hadoop.fs.s3a.S3AFileSystem;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.Text;
@@ -53,12 +54,14 @@
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.deployService;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestPropertyInt;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.terminateService;
 import static org.apache.hadoop.fs.s3a.auth.RoleTestUtils.probeForAssumedRoleARN;
 import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.*;
 import static org.apache.hadoop.fs.s3a.auth.delegation.MiniKerberizedHadoopCluster.assertSecurityEnabled;
 import static org.apache.hadoop.fs.s3a.auth.delegation.MiniKerberizedHadoopCluster.closeUserFileSystems;
 import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.getOrcData;
+import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.isUsingDefaultExternalDataFile;
 import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.requireAnonymousDataPath;
 
 /**
@@ -251,6 +254,10 @@ public void testCommonCrawlLookup() throws Throwable {
   public void testJobSubmissionCollectsTokens() throws Exception {
     describe("Mock Job test");
     JobConf conf = new JobConf(getConfiguration());
+    if (isUsingDefaultExternalDataFile(conf)) {
+      removeBaseAndBucketOverrides(conf,
+          Constants.ENDPOINT);
+    }
 
     // the input here is the external file; which lets
     // us differentiate source URI from dest URI

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java

@@ -118,12 +118,12 @@ protected int run(Object... args) throws Exception {
    * Run a S3GuardTool command from a varags list, catch any raised
    * ExitException and verify the status code matches that expected.
    * @param status expected status code of the exception
+   * @param conf configuration object.
    * @param args argument list
    * @throws Exception any exception
    */
-  protected void runToFailure(int status, Object... args)
+  protected void runToFailure(int status, Configuration conf, Object... args)
       throws Exception {
-    final Configuration conf = getConfiguration();
     ExitUtil.ExitException ex =
         intercept(ExitUtil.ExitException.class, () ->
             runS3GuardCommand(conf, args));
@@ -247,7 +247,7 @@ public void testUnsupported() throws Throwable {
     describe("Verify the unsupported tools are rejected");
     for (String tool : UNSUPPORTED_COMMANDS) {
       describe("Probing %s", tool);
-      runToFailure(E_S3GUARD_UNSUPPORTED, tool);
+      runToFailure(E_S3GUARD_UNSUPPORTED, getConfiguration(), tool);
     }
   }
 

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardTool.java

@@ -37,6 +37,7 @@
 import org.apache.hadoop.util.StringUtils;
 
 import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
+import static org.apache.hadoop.fs.s3a.Constants.ENDPOINT;
 import static org.apache.hadoop.fs.s3a.Constants.FIPS_ENDPOINT;
 import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_ALGORITHM;
 import static org.apache.hadoop.fs.s3a.MultipartTestUtils.assertNoUploadsAt;
@@ -48,6 +49,7 @@
 import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.E_BAD_STATE;
 import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.Uploads;
 import static org.apache.hadoop.fs.s3a.s3guard.S3GuardToolTestHelper.exec;
+import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.isUsingDefaultExternalDataFile;
 
 /**
  * Test S3Guard Tool CLI commands.
@@ -60,8 +62,13 @@ public class ITestS3GuardTool extends AbstractS3GuardToolTestBase {
 
   @Test
   public void testExternalBucketRequireUnencrypted() throws Throwable {
-    removeBaseAndBucketOverrides(getConfiguration(), S3_ENCRYPTION_ALGORITHM);
+    Configuration conf = getConfiguration();
-    run(BucketInfo.NAME,
+    if (isUsingDefaultExternalDataFile(conf)) {
+      removeBaseAndBucketOverrides(conf,
+          S3_ENCRYPTION_ALGORITHM,
+          ENDPOINT);
+    }
+    run(conf, BucketInfo.NAME,
         "-" + BucketInfo.ENCRYPTION_FLAG, "none",
         externalBucket());
   }
@@ -81,10 +88,17 @@ private String externalBucket() {
 
   @Test
   public void testExternalBucketRequireEncrypted() throws Throwable {
+    Configuration conf = getConfiguration();
+    if (isUsingDefaultExternalDataFile(conf)) {
+      removeBaseAndBucketOverrides(conf,
+          ENDPOINT);
+    }
     runToFailure(E_BAD_STATE,
+        conf,
         BucketInfo.NAME,
         "-" + BucketInfo.ENCRYPTION_FLAG,
-        "AES256", externalBucket());
+        "AES256",
+        externalBucket());
   }
 
   @Test
@@ -212,9 +226,13 @@ public void testUploadListByAge() throws Throwable {
 
   @Test
   public void testUploadNegativeExpect() throws Throwable {
-    runToFailure(E_BAD_STATE, Uploads.NAME, "-expect", "1",
+    Configuration conf = getConfiguration();
-        path("/we/are/almost/postive/this/doesnt/exist/fhfsadfoijew")
+    runToFailure(E_BAD_STATE,
-            .toString());
+        conf,
+        Uploads.NAME,
+        "-expect",
+        "1",
+        path("/we/are/almost/postive/this/doesnt/exist/fhfsadfoijew").toString());
   }
 
   private void assertNumUploads(Path path, int numUploads) throws Exception {

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java

@@ -60,6 +60,7 @@
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.assume;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.getInputStreamStatistics;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.getS3AInputStream;
+import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.isUsingDefaultExternalDataFile;
 import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.assertThatStatisticMinimum;
 import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.lookupMaximumStatistic;
 import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.lookupMeanStatistic;
@@ -99,7 +100,13 @@ public class ITestS3AInputStreamPerformance extends S3AScaleTestBase {
   @Override
   protected Configuration createScaleConfiguration() {
     Configuration conf = super.createScaleConfiguration();
-    S3ATestUtils.removeBaseAndBucketOverrides(conf, PREFETCH_ENABLED_KEY);
+    S3ATestUtils.removeBaseAndBucketOverrides(conf,
+        PREFETCH_ENABLED_KEY);
+    if (isUsingDefaultExternalDataFile(conf)) {
+      S3ATestUtils.removeBaseAndBucketOverrides(
+          conf,
+          ENDPOINT);
+    }
     conf.setBoolean(PREFETCH_ENABLED_KEY, false);
     return conf;
   }

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/PublicDatasetTestUtils.java

@@ -133,6 +133,18 @@ public static String requireDefaultExternalDataFile(Configuration conf) {
     return filename;
   }
 
+  /**
+   * To determine whether {@value S3ATestConstants#KEY_CSVTEST_FILE} is configured to be
+   * different from the default external file.
+   *
+   * @param conf Configuration object.
+   * @return True if the default external data file is being used.
+   */
+  public static boolean isUsingDefaultExternalDataFile(final Configuration conf) {
+    final String filename = getExternalData(conf).toUri().toString();
+    return DEFAULT_EXTERNAL_FILE.equals(filename);
+  }
+
   /**
    * Get the test external file; assume() that it is not modified (i.e. we haven't
    * switched to a new storage infrastructure where the bucket is no longer