From fa83c9a805041b94b3663b773e99e8074c534770 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Wed, 14 Aug 2024 02:57:44 -0700 Subject: [PATCH] HADOOP-19072 S3A: Override fs.s3a.performance.flags for tests (ADDENDUM 2) (#6993) Second followup to #6543; all hadoop-aws integration tests complete correctly even when fs.s3a.performance.flags = * Contributed by Viraj Jasani --- .../contract/s3a/ITestS3AContractCreate.java | 12 ++++-------- .../contract/s3a/ITestS3AContractMkdir.java | 14 ++++---------- .../ITestS3AContractMkdirWithCreatePerf.java | 13 +++---------- .../fs/s3a/ITestS3AFSMainOperations.java | 6 +++++- .../fs/s3a/ITestS3AFileOperationCost.java | 13 ++++--------- .../fs/s3a/ITestS3AFileSystemContract.java | 5 ++++- .../apache/hadoop/fs/s3a/S3ATestUtils.java | 19 +++++++++++++++++++ .../ITestS3AFileContextCreateMkdir.java | 12 ++++-------- ...stS3AFileContextCreateMkdirCreatePerf.java | 12 +++--------- .../ITestS3AFileContextMainOperations.java | 7 ++++++- .../fileContext/ITestS3AFileContextURI.java | 6 +++++- .../s3a/performance/ITestCreateFileCost.java | 11 ++++------- .../s3a/performance/ITestS3ADeleteCost.java | 13 ++++--------- 13 files changed, 69 insertions(+), 74 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java index a1067ddc0e..a6590e99e6 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java @@ -29,9 +29,7 @@ import org.apache.hadoop.fs.contract.AbstractFSContract; import org.apache.hadoop.fs.s3a.S3ATestUtils; -import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE; -import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS; -import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags; /** * S3A contract tests creating files. @@ -70,11 +68,9 @@ protected AbstractFSContract createContract(Configuration conf) { @Override protected Configuration createConfiguration() { - final Configuration conf = super.createConfiguration(); - removeBaseAndBucketOverrides(conf, - FS_S3A_CREATE_PERFORMANCE, - FS_S3A_PERFORMANCE_FLAGS); - conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, createPerformance); + final Configuration conf = setPerformanceFlags( + super.createConfiguration(), + createPerformance ? "create" : ""); S3ATestUtils.disableFilesystemCaching(conf); return conf; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdir.java index bce67ed67f..847f6980b5 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdir.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdir.java @@ -22,9 +22,7 @@ import org.apache.hadoop.fs.contract.AbstractContractMkdirTest; import org.apache.hadoop.fs.contract.AbstractFSContract; -import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE; -import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS; -import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags; /** * Test dir operations on S3A. @@ -33,13 +31,9 @@ public class ITestS3AContractMkdir extends AbstractContractMkdirTest { @Override protected Configuration createConfiguration() { - Configuration conf = super.createConfiguration(); - removeBaseAndBucketOverrides( - conf, - FS_S3A_CREATE_PERFORMANCE, - FS_S3A_PERFORMANCE_FLAGS); - conf.set(FS_S3A_PERFORMANCE_FLAGS, ""); - return conf; + return setPerformanceFlags( + super.createConfiguration(), + ""); } @Override diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdirWithCreatePerf.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdirWithCreatePerf.java index cacd6945d2..4b2468de97 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdirWithCreatePerf.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdirWithCreatePerf.java @@ -29,9 +29,7 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile; import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; -import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE; -import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS; -import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags; /** * Test mkdir operations on S3A with create performance mode. @@ -40,14 +38,9 @@ public class ITestS3AContractMkdirWithCreatePerf extends AbstractContractMkdirTe @Override protected Configuration createConfiguration() { - Configuration conf = super.createConfiguration(); - removeBaseAndBucketOverrides( - conf, - FS_S3A_CREATE_PERFORMANCE, - FS_S3A_PERFORMANCE_FLAGS); - conf.setStrings(FS_S3A_PERFORMANCE_FLAGS, + return setPerformanceFlags( + super.createConfiguration(), "create,mkdir"); - return conf; } @Override diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFSMainOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFSMainOperations.java index 013ec901d0..0281c57f5c 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFSMainOperations.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFSMainOperations.java @@ -31,6 +31,7 @@ import static org.apache.hadoop.fs.s3a.S3ATestUtils.createTestPath; import static org.apache.hadoop.fs.s3a.S3ATestUtils.isCreatePerformanceEnabled; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags; /** * S3A Test suite for the FSMainOperationsBaseTest tests. @@ -46,7 +47,10 @@ public ITestS3AFSMainOperations() { @Override protected FileSystem createFileSystem() throws Exception { - contract = new S3AContract(new Configuration()); + Configuration conf = setPerformanceFlags( + new Configuration(), + ""); + contract = new S3AContract(conf); contract.init(); return contract.getTestFileSystem(); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java index aa46557e91..585317c8da 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java @@ -40,9 +40,7 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.*; -import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE; -import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS; -import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags; import static org.apache.hadoop.fs.s3a.Statistic.*; import static org.apache.hadoop.fs.s3a.performance.OperationCost.*; import static org.apache.hadoop.test.GenericTestUtils.getTestDir; @@ -80,12 +78,9 @@ public ITestS3AFileOperationCost( @Override public Configuration createConfiguration() { - final Configuration conf = super.createConfiguration(); - removeBaseAndBucketOverrides(conf, - FS_S3A_CREATE_PERFORMANCE, - FS_S3A_PERFORMANCE_FLAGS); - conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, isKeepingMarkers()); - return conf; + return setPerformanceFlags( + super.createConfiguration(), + isKeepingMarkers() ? "create" : ""); } /** diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java index 56827043c9..4808145765 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java @@ -35,6 +35,7 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.skip; import static org.apache.hadoop.fs.s3a.S3ATestUtils.isCreatePerformanceEnabled; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags; import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.junit.Assume.*; import static org.junit.Assert.*; @@ -65,7 +66,9 @@ protected int getGlobalTimeout() { @Before public void setUp() throws Exception { nameThread(); - Configuration conf = new Configuration(); + Configuration conf = setPerformanceFlags( + new Configuration(), + ""); fs = S3ATestUtils.createTestFileSystem(conf); assumeNotNull(fs); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java index d8bb5898aa..3a3f875f5f 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java @@ -1127,6 +1127,25 @@ public static void assumeStoreAwsHosted(final FileSystem fs) { .getTrimmed(ENDPOINT, DEFAULT_ENDPOINT))); } + /** + * Modify the config by setting the performance flags and return the modified config. + * + * @param conf The configuration object. + * @param flagStr The performance flag string. + * @return The modified configuration object. + */ + public static Configuration setPerformanceFlags(final Configuration conf, + final String flagStr) { + removeBaseAndBucketOverrides( + conf, + FS_S3A_CREATE_PERFORMANCE, + FS_S3A_PERFORMANCE_FLAGS); + if (flagStr != null) { + conf.set(FS_S3A_PERFORMANCE_FLAGS, flagStr); + } + return conf; + } + /** * Helper class to do diffs of metrics. */ diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdir.java index ce7e9d4a43..095d2239ee 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdir.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdir.java @@ -18,9 +18,7 @@ import org.apache.hadoop.fs.s3a.S3ATestUtils; import org.junit.Before; -import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE; -import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS; -import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags; /** * Extends FileContextCreateMkdirBaseTest for a S3a FileContext. @@ -30,11 +28,9 @@ public class ITestS3AFileContextCreateMkdir @Before public void setUp() throws Exception { - Configuration conf = new Configuration(); - removeBaseAndBucketOverrides( - conf, - FS_S3A_CREATE_PERFORMANCE, - FS_S3A_PERFORMANCE_FLAGS); + Configuration conf = setPerformanceFlags( + new Configuration(), + null); fc = S3ATestUtils.createTestFileContext(conf); super.setUp(); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdirCreatePerf.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdirCreatePerf.java index 64039e4c52..68dde70bfe 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdirCreatePerf.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextCreateMkdirCreatePerf.java @@ -20,9 +20,7 @@ import org.apache.hadoop.fs.FileContextCreateMkdirBaseTest; import org.apache.hadoop.fs.s3a.S3ATestUtils; -import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE; -import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS; -import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags; import static org.apache.hadoop.test.LambdaTestUtils.intercept; /** @@ -34,12 +32,8 @@ public class ITestS3AFileContextCreateMkdirCreatePerf @Before public void setUp() throws Exception { - Configuration conf = new Configuration(); - removeBaseAndBucketOverrides( - conf, - FS_S3A_CREATE_PERFORMANCE, - FS_S3A_PERFORMANCE_FLAGS); - conf.setStrings(FS_S3A_PERFORMANCE_FLAGS, + Configuration conf = setPerformanceFlags( + new Configuration(), "mkdir"); fc = S3ATestUtils.createTestFileContext(conf); super.setUp(); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextMainOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextMainOperations.java index b28f88e43b..cc630484a1 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextMainOperations.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextMainOperations.java @@ -27,6 +27,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.s3a.S3ATestUtils; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags; + /** * S3A implementation of FileContextMainOperationsBaseTest. */ @@ -36,7 +38,10 @@ public class ITestS3AFileContextMainOperations @Before public void setUp() throws IOException, Exception { - Configuration conf = new Configuration(); + Configuration conf = setPerformanceFlags( + new Configuration(), + ""); + fc = S3ATestUtils.createTestFileContext(conf); super.setUp(); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java index bef359cca7..54161d1012 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextURI.java @@ -21,6 +21,8 @@ import org.junit.Ignore; import org.junit.Test; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags; + /** * S3a implementation of FileContextURIBase. */ @@ -30,7 +32,9 @@ public class ITestS3AFileContextURI extends FileContextURIBase { @Before public void setUp() throws IOException, Exception { - conf = new Configuration(); + conf = setPerformanceFlags( + new Configuration(), + ""); fc1 = S3ATestUtils.createTestFileContext(conf); fc2 = S3ATestUtils.createTestFileContext(conf); //different object, same FS diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestCreateFileCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestCreateFileCost.java index 5bd4bf412f..65786bf6d6 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestCreateFileCost.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestCreateFileCost.java @@ -42,9 +42,8 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.toChar; import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_HEADER; import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE; -import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS; import static org.apache.hadoop.fs.s3a.Constants.XA_HEADER_PREFIX; -import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags; import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_BULK_DELETE_REQUEST; import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_DELETE_REQUEST; import static org.apache.hadoop.fs.s3a.performance.OperationCost.CREATE_FILE_NO_OVERWRITE; @@ -105,11 +104,9 @@ private OperationCost expected(OperationCost source) { @Override public Configuration createConfiguration() { - final Configuration conf = super.createConfiguration(); - removeBaseAndBucketOverrides(conf, - FS_S3A_CREATE_PERFORMANCE, - FS_S3A_PERFORMANCE_FLAGS); - conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, createPerformance); + final Configuration conf = setPerformanceFlags( + super.createConfiguration(), + createPerformance ? "create" : ""); S3ATestUtils.disableFilesystemCaching(conf); return conf; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java index 9979b72e71..6bd4114f07 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java @@ -39,9 +39,7 @@ import org.apache.hadoop.fs.s3a.Tristate; import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum; -import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE; -import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS; -import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags; import static org.apache.hadoop.fs.s3a.Statistic.*; import static org.apache.hadoop.fs.s3a.performance.OperationCost.*; import static org.apache.hadoop.fs.s3a.performance.OperationCostValidator.probe; @@ -80,12 +78,9 @@ public ITestS3ADeleteCost(final String name, @Override public Configuration createConfiguration() { - Configuration conf = super.createConfiguration(); - removeBaseAndBucketOverrides(conf, - FS_S3A_CREATE_PERFORMANCE, - FS_S3A_PERFORMANCE_FLAGS); - conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, false); - return conf; + return setPerformanceFlags( + super.createConfiguration(), + ""); } @Override