big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-19146. S3A: noaa-cors-pds test bucket access with global endpoint fails (#6723)

Browse Source 
HADOOP-19057 switched the hadoop-aws test bucket from landsat-pds to 
noaa-cors-pds 

This new bucket isn't accessible if the client configuration
sets an fs.s3a.endpoint/region value other than us-east-1.

Contributed by Viraj Jasani
This commit is contained in:
Viraj Jasani 2024-04-30 03:16:36 -08:00 committed by GitHub
parent 910cb6b887
commit a8a58944bd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 72 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java
View File

@ -44,6 +44,7 @@
import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.DELEGATION_TOKEN_BINDING;
import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.CONSTRUCTOR_EXCEPTION;
import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.getExternalData;
import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.isUsingDefaultExternalDataFile;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
import static org.junit.Assert.*;
@ -207,6 +208,11 @@ public void testBadCredentialsWithRemap() throws Exception {
@Test
public void testAnonymousProvider() throws Exception {
Configuration conf = createConf(AnonymousAWSCredentialsProvider.class);
if (isUsingDefaultExternalDataFile(conf)) {
removeBaseAndBucketOverrides(conf,
ENDPOINT);
conf.set(ENDPOINT, CENTRAL_ENDPOINT);
}
Path testFile = getExternalData(conf);
try (FileSystem fs = FileSystem.newInstance(testFile.toUri(), conf)) {
Assertions.assertThat(fs)

10
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java
View File

@ -32,9 +32,6 @@
import org.apache.hadoop.fs.store.audit.AuditSpan;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
@ -45,7 +42,9 @@
import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.createFiles;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.isBulkDeleteEnabled;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.apache.hadoop.fs.s3a.test.ExtraAssertions.failIf;
import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.isUsingDefaultExternalDataFile;
import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.requireDefaultExternalData;
import static org.apache.hadoop.test.LambdaTestUtils.*;
import static org.apache.hadoop.util.functional.RemoteIterators.mappingRemoteIterator;
@ -55,14 +54,15 @@
* ITest for failure handling, primarily multipart deletion.
*/
public class ITestS3AFailureHandling extends AbstractS3ATestBase {
private static final Logger LOG =
LoggerFactory.getLogger(ITestS3AFailureHandling.class);
@Override
protected Configuration createConfiguration() {
Configuration conf = super.createConfiguration();
S3ATestUtils.disableFilesystemCaching(conf);
conf.setBoolean(Constants.ENABLE_MULTI_DELETE, true);
if (isUsingDefaultExternalDataFile(conf)) {
removeBaseAndBucketOverrides(conf, Constants.ENDPOINT);
}
return conf;
}

8
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3APrefetchingCacheFiles.java
View File

@ -38,9 +38,11 @@
import org.apache.hadoop.fs.s3a.performance.AbstractS3ACostTest;
import static org.apache.hadoop.fs.s3a.Constants.BUFFER_DIR;
import static org.apache.hadoop.fs.s3a.Constants.ENDPOINT;
import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_BLOCK_SIZE_KEY;
import static org.apache.hadoop.fs.s3a.Constants.PREFETCH_ENABLED_KEY;
import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.getExternalData;
import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.isUsingDefaultExternalDataFile;
import static org.apache.hadoop.io.IOUtils.cleanupWithLogger;
/**
@ -84,7 +86,11 @@ public void setUp() throws Exception {
@Override
public Configuration createConfiguration() {
Configuration configuration = super.createConfiguration();
S3ATestUtils.removeBaseAndBucketOverrides(configuration, PREFETCH_ENABLED_KEY);
if (isUsingDefaultExternalDataFile(configuration)) {
S3ATestUtils.removeBaseAndBucketOverrides(configuration,
PREFETCH_ENABLED_KEY,
ENDPOINT);
}
configuration.setBoolean(PREFETCH_ENABLED_KEY, true);
// use a small block size unless explicitly set in the test config.
configuration.setInt(PREFETCH_BLOCK_SIZE_KEY, BLOCK_SIZE);

7
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestDelegatedMRJob.java
View File

@ -33,6 +33,7 @@
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.s3a.Constants;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
@ -53,12 +54,14 @@
import static org.apache.hadoop.fs.s3a.S3ATestUtils.deployService;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestPropertyInt;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.terminateService;
import static org.apache.hadoop.fs.s3a.auth.RoleTestUtils.probeForAssumedRoleARN;
import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.*;
import static org.apache.hadoop.fs.s3a.auth.delegation.MiniKerberizedHadoopCluster.assertSecurityEnabled;
import static org.apache.hadoop.fs.s3a.auth.delegation.MiniKerberizedHadoopCluster.closeUserFileSystems;
import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.getOrcData;
import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.isUsingDefaultExternalDataFile;
import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.requireAnonymousDataPath;
/**
@ -251,6 +254,10 @@ public void testCommonCrawlLookup() throws Throwable {
public void testJobSubmissionCollectsTokens() throws Exception {
describe("Mock Job test");
JobConf conf = new JobConf(getConfiguration());
if (isUsingDefaultExternalDataFile(conf)) {
removeBaseAndBucketOverrides(conf,
Constants.ENDPOINT);
}
// the input here is the external file; which lets
// us differentiate source URI from dest URI

6
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
View File

@ -118,12 +118,12 @@ protected int run(Object... args) throws Exception {
* Run a S3GuardTool command from a varags list, catch any raised
* ExitException and verify the status code matches that expected.
* @param status expected status code of the exception
* @param conf configuration object.
* @param args argument list
* @throws Exception any exception
*/
protected void runToFailure(int status, Object... args)
protected void runToFailure(int status, Configuration conf, Object... args)
throws Exception {
final Configuration conf = getConfiguration();
ExitUtil.ExitException ex =
intercept(ExitUtil.ExitException.class, () ->
runS3GuardCommand(conf, args));
@ -247,7 +247,7 @@ public void testUnsupported() throws Throwable {
describe("Verify the unsupported tools are rejected");
for (String tool : UNSUPPORTED_COMMANDS) {
describe("Probing %s", tool);
runToFailure(E_S3GUARD_UNSUPPORTED, tool);
runToFailure(E_S3GUARD_UNSUPPORTED, getConfiguration(), tool);
}
}

30
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardTool.java
View File

@ -37,6 +37,7 @@
import org.apache.hadoop.util.StringUtils;
import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
import static org.apache.hadoop.fs.s3a.Constants.ENDPOINT;
import static org.apache.hadoop.fs.s3a.Constants.FIPS_ENDPOINT;
import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_ALGORITHM;
import static org.apache.hadoop.fs.s3a.MultipartTestUtils.assertNoUploadsAt;
@ -48,6 +49,7 @@
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.E_BAD_STATE;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.Uploads;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardToolTestHelper.exec;
import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.isUsingDefaultExternalDataFile;
/**
* Test S3Guard Tool CLI commands.
@ -60,8 +62,13 @@ public class ITestS3GuardTool extends AbstractS3GuardToolTestBase {
@Test
public void testExternalBucketRequireUnencrypted() throws Throwable {
removeBaseAndBucketOverrides(getConfiguration(), S3_ENCRYPTION_ALGORITHM);
run(BucketInfo.NAME,
Configuration conf = getConfiguration();
if (isUsingDefaultExternalDataFile(conf)) {
removeBaseAndBucketOverrides(conf,
S3_ENCRYPTION_ALGORITHM,
ENDPOINT);
}
run(conf, BucketInfo.NAME,
"-" + BucketInfo.ENCRYPTION_FLAG, "none",
externalBucket());
}
@ -81,10 +88,17 @@ private String externalBucket() {
@Test
public void testExternalBucketRequireEncrypted() throws Throwable {
Configuration conf = getConfiguration();
if (isUsingDefaultExternalDataFile(conf)) {
removeBaseAndBucketOverrides(conf,
ENDPOINT);
}
runToFailure(E_BAD_STATE,
conf,
BucketInfo.NAME,
"-" + BucketInfo.ENCRYPTION_FLAG,
"AES256", externalBucket());
"AES256",
externalBucket());
}
@Test
@ -212,9 +226,13 @@ public void testUploadListByAge() throws Throwable {
@Test
public void testUploadNegativeExpect() throws Throwable {
runToFailure(E_BAD_STATE, Uploads.NAME, "-expect", "1",
path("/we/are/almost/postive/this/doesnt/exist/fhfsadfoijew")
.toString());
Configuration conf = getConfiguration();
runToFailure(E_BAD_STATE,
conf,
Uploads.NAME,
"-expect",
"1",
path("/we/are/almost/postive/this/doesnt/exist/fhfsadfoijew").toString());
}
private void assertNumUploads(Path path, int numUploads) throws Exception {

9
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java
View File

@ -60,6 +60,7 @@
import static org.apache.hadoop.fs.s3a.S3ATestUtils.assume;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.getInputStreamStatistics;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.getS3AInputStream;
import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.isUsingDefaultExternalDataFile;
import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.assertThatStatisticMinimum;
import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.lookupMaximumStatistic;
import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.lookupMeanStatistic;
@ -99,7 +100,13 @@ public class ITestS3AInputStreamPerformance extends S3AScaleTestBase {
@Override
protected Configuration createScaleConfiguration() {
Configuration conf = super.createScaleConfiguration();
S3ATestUtils.removeBaseAndBucketOverrides(conf, PREFETCH_ENABLED_KEY);
S3ATestUtils.removeBaseAndBucketOverrides(conf,
PREFETCH_ENABLED_KEY);
if (isUsingDefaultExternalDataFile(conf)) {
S3ATestUtils.removeBaseAndBucketOverrides(
conf,
ENDPOINT);
}
conf.setBoolean(PREFETCH_ENABLED_KEY, false);
return conf;
}

12
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/PublicDatasetTestUtils.java
View File

@ -133,6 +133,18 @@ public static String requireDefaultExternalDataFile(Configuration conf) {
return filename;
}
/**
* To determine whether {@value S3ATestConstants#KEY_CSVTEST_FILE} is configured to be
* different from the default external file.
*
* @param conf Configuration object.
* @return True if the default external data file is being used.
*/
public static boolean isUsingDefaultExternalDataFile(final Configuration conf) {
final String filename = getExternalData(conf).toUri().toString();
return DEFAULT_EXTERNAL_FILE.equals(filename);
}
/**
* Get the test external file; assume() that it is not modified (i.e. we haven't
* switched to a new storage infrastructure where the bucket is no longer