HADOOP-19072 S3A: Override fs.s3a.performance.flags for tests (ADDENDUM 2) (#6993)

Second followup to #6543; all hadoop-aws integration tests complete correctly even when 

fs.s3a.performance.flags = *

Contributed by Viraj Jasani
This commit is contained in:
Viraj Jasani 2024-08-14 02:57:44 -07:00 committed by GitHub
parent 74ff00705c
commit fa83c9a805
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 69 additions and 74 deletions

View File

@ -29,9 +29,7 @@
import org.apache.hadoop.fs.contract.AbstractFSContract; import org.apache.hadoop.fs.contract.AbstractFSContract;
import org.apache.hadoop.fs.s3a.S3ATestUtils; import org.apache.hadoop.fs.s3a.S3ATestUtils;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE; import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
/** /**
* S3A contract tests creating files. * S3A contract tests creating files.
@ -70,11 +68,9 @@ protected AbstractFSContract createContract(Configuration conf) {
@Override @Override
protected Configuration createConfiguration() { protected Configuration createConfiguration() {
final Configuration conf = super.createConfiguration(); final Configuration conf = setPerformanceFlags(
removeBaseAndBucketOverrides(conf, super.createConfiguration(),
FS_S3A_CREATE_PERFORMANCE, createPerformance ? "create" : "");
FS_S3A_PERFORMANCE_FLAGS);
conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, createPerformance);
S3ATestUtils.disableFilesystemCaching(conf); S3ATestUtils.disableFilesystemCaching(conf);
return conf; return conf;
} }

View File

@ -22,9 +22,7 @@
import org.apache.hadoop.fs.contract.AbstractContractMkdirTest; import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
import org.apache.hadoop.fs.contract.AbstractFSContract; import org.apache.hadoop.fs.contract.AbstractFSContract;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE; import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
/** /**
* Test dir operations on S3A. * Test dir operations on S3A.
@ -33,13 +31,9 @@ public class ITestS3AContractMkdir extends AbstractContractMkdirTest {
@Override @Override
protected Configuration createConfiguration() { protected Configuration createConfiguration() {
Configuration conf = super.createConfiguration(); return setPerformanceFlags(
removeBaseAndBucketOverrides( super.createConfiguration(),
conf, "");
FS_S3A_CREATE_PERFORMANCE,
FS_S3A_PERFORMANCE_FLAGS);
conf.set(FS_S3A_PERFORMANCE_FLAGS, "");
return conf;
} }
@Override @Override

View File

@ -29,9 +29,7 @@
import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile; import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE; import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
/** /**
* Test mkdir operations on S3A with create performance mode. * Test mkdir operations on S3A with create performance mode.
@ -40,14 +38,9 @@ public class ITestS3AContractMkdirWithCreatePerf extends AbstractContractMkdirTe
@Override @Override
protected Configuration createConfiguration() { protected Configuration createConfiguration() {
Configuration conf = super.createConfiguration(); return setPerformanceFlags(
removeBaseAndBucketOverrides( super.createConfiguration(),
conf,
FS_S3A_CREATE_PERFORMANCE,
FS_S3A_PERFORMANCE_FLAGS);
conf.setStrings(FS_S3A_PERFORMANCE_FLAGS,
"create,mkdir"); "create,mkdir");
return conf;
} }
@Override @Override

View File

@ -31,6 +31,7 @@
import static org.apache.hadoop.fs.s3a.S3ATestUtils.createTestPath; import static org.apache.hadoop.fs.s3a.S3ATestUtils.createTestPath;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.isCreatePerformanceEnabled; import static org.apache.hadoop.fs.s3a.S3ATestUtils.isCreatePerformanceEnabled;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
/** /**
* S3A Test suite for the FSMainOperationsBaseTest tests. * S3A Test suite for the FSMainOperationsBaseTest tests.
@ -46,7 +47,10 @@ public ITestS3AFSMainOperations() {
@Override @Override
protected FileSystem createFileSystem() throws Exception { protected FileSystem createFileSystem() throws Exception {
contract = new S3AContract(new Configuration()); Configuration conf = setPerformanceFlags(
new Configuration(),
"");
contract = new S3AContract(conf);
contract.init(); contract.init();
return contract.getTestFileSystem(); return contract.getTestFileSystem();
} }

View File

@ -40,9 +40,7 @@
import static org.apache.hadoop.fs.contract.ContractTestUtils.*; import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE; import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.apache.hadoop.fs.s3a.Statistic.*; import static org.apache.hadoop.fs.s3a.Statistic.*;
import static org.apache.hadoop.fs.s3a.performance.OperationCost.*; import static org.apache.hadoop.fs.s3a.performance.OperationCost.*;
import static org.apache.hadoop.test.GenericTestUtils.getTestDir; import static org.apache.hadoop.test.GenericTestUtils.getTestDir;
@ -80,12 +78,9 @@ public ITestS3AFileOperationCost(
@Override @Override
public Configuration createConfiguration() { public Configuration createConfiguration() {
final Configuration conf = super.createConfiguration(); return setPerformanceFlags(
removeBaseAndBucketOverrides(conf, super.createConfiguration(),
FS_S3A_CREATE_PERFORMANCE, isKeepingMarkers() ? "create" : "");
FS_S3A_PERFORMANCE_FLAGS);
conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, isKeepingMarkers());
return conf;
} }
/** /**

View File

@ -35,6 +35,7 @@
import static org.apache.hadoop.fs.contract.ContractTestUtils.skip; import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.isCreatePerformanceEnabled; import static org.apache.hadoop.fs.s3a.S3ATestUtils.isCreatePerformanceEnabled;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.apache.hadoop.test.LambdaTestUtils.intercept;
import static org.junit.Assume.*; import static org.junit.Assume.*;
import static org.junit.Assert.*; import static org.junit.Assert.*;
@ -65,7 +66,9 @@ protected int getGlobalTimeout() {
@Before @Before
public void setUp() throws Exception { public void setUp() throws Exception {
nameThread(); nameThread();
Configuration conf = new Configuration(); Configuration conf = setPerformanceFlags(
new Configuration(),
"");
fs = S3ATestUtils.createTestFileSystem(conf); fs = S3ATestUtils.createTestFileSystem(conf);
assumeNotNull(fs); assumeNotNull(fs);

View File

@ -1127,6 +1127,25 @@ public static void assumeStoreAwsHosted(final FileSystem fs) {
.getTrimmed(ENDPOINT, DEFAULT_ENDPOINT))); .getTrimmed(ENDPOINT, DEFAULT_ENDPOINT)));
} }
/**
* Modify the config by setting the performance flags and return the modified config.
*
* @param conf The configuration object.
* @param flagStr The performance flag string.
* @return The modified configuration object.
*/
public static Configuration setPerformanceFlags(final Configuration conf,
final String flagStr) {
removeBaseAndBucketOverrides(
conf,
FS_S3A_CREATE_PERFORMANCE,
FS_S3A_PERFORMANCE_FLAGS);
if (flagStr != null) {
conf.set(FS_S3A_PERFORMANCE_FLAGS, flagStr);
}
return conf;
}
/** /**
* Helper class to do diffs of metrics. * Helper class to do diffs of metrics.
*/ */

View File

@ -18,9 +18,7 @@
import org.apache.hadoop.fs.s3a.S3ATestUtils; import org.apache.hadoop.fs.s3a.S3ATestUtils;
import org.junit.Before; import org.junit.Before;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE; import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
/** /**
* Extends FileContextCreateMkdirBaseTest for a S3a FileContext. * Extends FileContextCreateMkdirBaseTest for a S3a FileContext.
@ -30,11 +28,9 @@ public class ITestS3AFileContextCreateMkdir
@Before @Before
public void setUp() throws Exception { public void setUp() throws Exception {
Configuration conf = new Configuration(); Configuration conf = setPerformanceFlags(
removeBaseAndBucketOverrides( new Configuration(),
conf, null);
FS_S3A_CREATE_PERFORMANCE,
FS_S3A_PERFORMANCE_FLAGS);
fc = S3ATestUtils.createTestFileContext(conf); fc = S3ATestUtils.createTestFileContext(conf);
super.setUp(); super.setUp();
} }

View File

@ -20,9 +20,7 @@
import org.apache.hadoop.fs.FileContextCreateMkdirBaseTest; import org.apache.hadoop.fs.FileContextCreateMkdirBaseTest;
import org.apache.hadoop.fs.s3a.S3ATestUtils; import org.apache.hadoop.fs.s3a.S3ATestUtils;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE; import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.apache.hadoop.test.LambdaTestUtils.intercept;
/** /**
@ -34,12 +32,8 @@ public class ITestS3AFileContextCreateMkdirCreatePerf
@Before @Before
public void setUp() throws Exception { public void setUp() throws Exception {
Configuration conf = new Configuration(); Configuration conf = setPerformanceFlags(
removeBaseAndBucketOverrides( new Configuration(),
conf,
FS_S3A_CREATE_PERFORMANCE,
FS_S3A_PERFORMANCE_FLAGS);
conf.setStrings(FS_S3A_PERFORMANCE_FLAGS,
"mkdir"); "mkdir");
fc = S3ATestUtils.createTestFileContext(conf); fc = S3ATestUtils.createTestFileContext(conf);
super.setUp(); super.setUp();

View File

@ -27,6 +27,8 @@
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.s3a.S3ATestUtils; import org.apache.hadoop.fs.s3a.S3ATestUtils;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
/** /**
* S3A implementation of FileContextMainOperationsBaseTest. * S3A implementation of FileContextMainOperationsBaseTest.
*/ */
@ -36,7 +38,10 @@ public class ITestS3AFileContextMainOperations
@Before @Before
public void setUp() throws IOException, Exception { public void setUp() throws IOException, Exception {
Configuration conf = new Configuration(); Configuration conf = setPerformanceFlags(
new Configuration(),
"");
fc = S3ATestUtils.createTestFileContext(conf); fc = S3ATestUtils.createTestFileContext(conf);
super.setUp(); super.setUp();
} }

View File

@ -21,6 +21,8 @@
import org.junit.Ignore; import org.junit.Ignore;
import org.junit.Test; import org.junit.Test;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
/** /**
* S3a implementation of FileContextURIBase. * S3a implementation of FileContextURIBase.
*/ */
@ -30,7 +32,9 @@ public class ITestS3AFileContextURI extends FileContextURIBase {
@Before @Before
public void setUp() throws IOException, Exception { public void setUp() throws IOException, Exception {
conf = new Configuration(); conf = setPerformanceFlags(
new Configuration(),
"");
fc1 = S3ATestUtils.createTestFileContext(conf); fc1 = S3ATestUtils.createTestFileContext(conf);
fc2 = S3ATestUtils.createTestFileContext(conf); //different object, same FS fc2 = S3ATestUtils.createTestFileContext(conf); //different object, same FS

View File

@ -42,9 +42,8 @@
import static org.apache.hadoop.fs.contract.ContractTestUtils.toChar; import static org.apache.hadoop.fs.contract.ContractTestUtils.toChar;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_HEADER; import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_HEADER;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE; import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
import static org.apache.hadoop.fs.s3a.Constants.XA_HEADER_PREFIX; import static org.apache.hadoop.fs.s3a.Constants.XA_HEADER_PREFIX;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_BULK_DELETE_REQUEST; import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_BULK_DELETE_REQUEST;
import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_DELETE_REQUEST; import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_DELETE_REQUEST;
import static org.apache.hadoop.fs.s3a.performance.OperationCost.CREATE_FILE_NO_OVERWRITE; import static org.apache.hadoop.fs.s3a.performance.OperationCost.CREATE_FILE_NO_OVERWRITE;
@ -105,11 +104,9 @@ private OperationCost expected(OperationCost source) {
@Override @Override
public Configuration createConfiguration() { public Configuration createConfiguration() {
final Configuration conf = super.createConfiguration(); final Configuration conf = setPerformanceFlags(
removeBaseAndBucketOverrides(conf, super.createConfiguration(),
FS_S3A_CREATE_PERFORMANCE, createPerformance ? "create" : "");
FS_S3A_PERFORMANCE_FLAGS);
conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, createPerformance);
S3ATestUtils.disableFilesystemCaching(conf); S3ATestUtils.disableFilesystemCaching(conf);
return conf; return conf;
} }

View File

@ -39,9 +39,7 @@
import org.apache.hadoop.fs.s3a.Tristate; import org.apache.hadoop.fs.s3a.Tristate;
import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum; import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE; import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
import static org.apache.hadoop.fs.s3a.Statistic.*; import static org.apache.hadoop.fs.s3a.Statistic.*;
import static org.apache.hadoop.fs.s3a.performance.OperationCost.*; import static org.apache.hadoop.fs.s3a.performance.OperationCost.*;
import static org.apache.hadoop.fs.s3a.performance.OperationCostValidator.probe; import static org.apache.hadoop.fs.s3a.performance.OperationCostValidator.probe;
@ -80,12 +78,9 @@ public ITestS3ADeleteCost(final String name,
@Override @Override
public Configuration createConfiguration() { public Configuration createConfiguration() {
Configuration conf = super.createConfiguration(); return setPerformanceFlags(
removeBaseAndBucketOverrides(conf, super.createConfiguration(),
FS_S3A_CREATE_PERFORMANCE, "");
FS_S3A_PERFORMANCE_FLAGS);
conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, false);
return conf;
} }
@Override @Override