HADOOP-19072 S3A: Override fs.s3a.performance.flags for tests (ADDENDUM 2) (#6993)
Second followup to #6543; all hadoop-aws integration tests complete correctly even when fs.s3a.performance.flags = * Contributed by Viraj Jasani
This commit is contained in:
parent
74ff00705c
commit
fa83c9a805
@ -29,9 +29,7 @@
|
|||||||
import org.apache.hadoop.fs.contract.AbstractFSContract;
|
import org.apache.hadoop.fs.contract.AbstractFSContract;
|
||||||
import org.apache.hadoop.fs.s3a.S3ATestUtils;
|
import org.apache.hadoop.fs.s3a.S3ATestUtils;
|
||||||
|
|
||||||
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
|
import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
|
||||||
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
|
|
||||||
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* S3A contract tests creating files.
|
* S3A contract tests creating files.
|
||||||
@ -70,11 +68,9 @@ protected AbstractFSContract createContract(Configuration conf) {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected Configuration createConfiguration() {
|
protected Configuration createConfiguration() {
|
||||||
final Configuration conf = super.createConfiguration();
|
final Configuration conf = setPerformanceFlags(
|
||||||
removeBaseAndBucketOverrides(conf,
|
super.createConfiguration(),
|
||||||
FS_S3A_CREATE_PERFORMANCE,
|
createPerformance ? "create" : "");
|
||||||
FS_S3A_PERFORMANCE_FLAGS);
|
|
||||||
conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, createPerformance);
|
|
||||||
S3ATestUtils.disableFilesystemCaching(conf);
|
S3ATestUtils.disableFilesystemCaching(conf);
|
||||||
return conf;
|
return conf;
|
||||||
}
|
}
|
||||||
|
@ -22,9 +22,7 @@
|
|||||||
import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
|
import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
|
||||||
import org.apache.hadoop.fs.contract.AbstractFSContract;
|
import org.apache.hadoop.fs.contract.AbstractFSContract;
|
||||||
|
|
||||||
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
|
import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
|
||||||
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
|
|
||||||
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Test dir operations on S3A.
|
* Test dir operations on S3A.
|
||||||
@ -33,13 +31,9 @@ public class ITestS3AContractMkdir extends AbstractContractMkdirTest {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected Configuration createConfiguration() {
|
protected Configuration createConfiguration() {
|
||||||
Configuration conf = super.createConfiguration();
|
return setPerformanceFlags(
|
||||||
removeBaseAndBucketOverrides(
|
super.createConfiguration(),
|
||||||
conf,
|
"");
|
||||||
FS_S3A_CREATE_PERFORMANCE,
|
|
||||||
FS_S3A_PERFORMANCE_FLAGS);
|
|
||||||
conf.set(FS_S3A_PERFORMANCE_FLAGS, "");
|
|
||||||
return conf;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -29,9 +29,7 @@
|
|||||||
|
|
||||||
import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
|
import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
|
||||||
import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
|
import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
|
||||||
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
|
import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
|
||||||
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
|
|
||||||
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Test mkdir operations on S3A with create performance mode.
|
* Test mkdir operations on S3A with create performance mode.
|
||||||
@ -40,14 +38,9 @@ public class ITestS3AContractMkdirWithCreatePerf extends AbstractContractMkdirTe
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected Configuration createConfiguration() {
|
protected Configuration createConfiguration() {
|
||||||
Configuration conf = super.createConfiguration();
|
return setPerformanceFlags(
|
||||||
removeBaseAndBucketOverrides(
|
super.createConfiguration(),
|
||||||
conf,
|
|
||||||
FS_S3A_CREATE_PERFORMANCE,
|
|
||||||
FS_S3A_PERFORMANCE_FLAGS);
|
|
||||||
conf.setStrings(FS_S3A_PERFORMANCE_FLAGS,
|
|
||||||
"create,mkdir");
|
"create,mkdir");
|
||||||
return conf;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -31,6 +31,7 @@
|
|||||||
|
|
||||||
import static org.apache.hadoop.fs.s3a.S3ATestUtils.createTestPath;
|
import static org.apache.hadoop.fs.s3a.S3ATestUtils.createTestPath;
|
||||||
import static org.apache.hadoop.fs.s3a.S3ATestUtils.isCreatePerformanceEnabled;
|
import static org.apache.hadoop.fs.s3a.S3ATestUtils.isCreatePerformanceEnabled;
|
||||||
|
import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* S3A Test suite for the FSMainOperationsBaseTest tests.
|
* S3A Test suite for the FSMainOperationsBaseTest tests.
|
||||||
@ -46,7 +47,10 @@ public ITestS3AFSMainOperations() {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected FileSystem createFileSystem() throws Exception {
|
protected FileSystem createFileSystem() throws Exception {
|
||||||
contract = new S3AContract(new Configuration());
|
Configuration conf = setPerformanceFlags(
|
||||||
|
new Configuration(),
|
||||||
|
"");
|
||||||
|
contract = new S3AContract(conf);
|
||||||
contract.init();
|
contract.init();
|
||||||
return contract.getTestFileSystem();
|
return contract.getTestFileSystem();
|
||||||
}
|
}
|
||||||
|
@ -40,9 +40,7 @@
|
|||||||
|
|
||||||
|
|
||||||
import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
|
import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
|
||||||
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
|
import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
|
||||||
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
|
|
||||||
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
|
|
||||||
import static org.apache.hadoop.fs.s3a.Statistic.*;
|
import static org.apache.hadoop.fs.s3a.Statistic.*;
|
||||||
import static org.apache.hadoop.fs.s3a.performance.OperationCost.*;
|
import static org.apache.hadoop.fs.s3a.performance.OperationCost.*;
|
||||||
import static org.apache.hadoop.test.GenericTestUtils.getTestDir;
|
import static org.apache.hadoop.test.GenericTestUtils.getTestDir;
|
||||||
@ -80,12 +78,9 @@ public ITestS3AFileOperationCost(
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Configuration createConfiguration() {
|
public Configuration createConfiguration() {
|
||||||
final Configuration conf = super.createConfiguration();
|
return setPerformanceFlags(
|
||||||
removeBaseAndBucketOverrides(conf,
|
super.createConfiguration(),
|
||||||
FS_S3A_CREATE_PERFORMANCE,
|
isKeepingMarkers() ? "create" : "");
|
||||||
FS_S3A_PERFORMANCE_FLAGS);
|
|
||||||
conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, isKeepingMarkers());
|
|
||||||
return conf;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -35,6 +35,7 @@
|
|||||||
|
|
||||||
import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
|
import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
|
||||||
import static org.apache.hadoop.fs.s3a.S3ATestUtils.isCreatePerformanceEnabled;
|
import static org.apache.hadoop.fs.s3a.S3ATestUtils.isCreatePerformanceEnabled;
|
||||||
|
import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
|
||||||
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
|
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
|
||||||
import static org.junit.Assume.*;
|
import static org.junit.Assume.*;
|
||||||
import static org.junit.Assert.*;
|
import static org.junit.Assert.*;
|
||||||
@ -65,7 +66,9 @@ protected int getGlobalTimeout() {
|
|||||||
@Before
|
@Before
|
||||||
public void setUp() throws Exception {
|
public void setUp() throws Exception {
|
||||||
nameThread();
|
nameThread();
|
||||||
Configuration conf = new Configuration();
|
Configuration conf = setPerformanceFlags(
|
||||||
|
new Configuration(),
|
||||||
|
"");
|
||||||
|
|
||||||
fs = S3ATestUtils.createTestFileSystem(conf);
|
fs = S3ATestUtils.createTestFileSystem(conf);
|
||||||
assumeNotNull(fs);
|
assumeNotNull(fs);
|
||||||
|
@ -1127,6 +1127,25 @@ public static void assumeStoreAwsHosted(final FileSystem fs) {
|
|||||||
.getTrimmed(ENDPOINT, DEFAULT_ENDPOINT)));
|
.getTrimmed(ENDPOINT, DEFAULT_ENDPOINT)));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Modify the config by setting the performance flags and return the modified config.
|
||||||
|
*
|
||||||
|
* @param conf The configuration object.
|
||||||
|
* @param flagStr The performance flag string.
|
||||||
|
* @return The modified configuration object.
|
||||||
|
*/
|
||||||
|
public static Configuration setPerformanceFlags(final Configuration conf,
|
||||||
|
final String flagStr) {
|
||||||
|
removeBaseAndBucketOverrides(
|
||||||
|
conf,
|
||||||
|
FS_S3A_CREATE_PERFORMANCE,
|
||||||
|
FS_S3A_PERFORMANCE_FLAGS);
|
||||||
|
if (flagStr != null) {
|
||||||
|
conf.set(FS_S3A_PERFORMANCE_FLAGS, flagStr);
|
||||||
|
}
|
||||||
|
return conf;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Helper class to do diffs of metrics.
|
* Helper class to do diffs of metrics.
|
||||||
*/
|
*/
|
||||||
|
@ -18,9 +18,7 @@
|
|||||||
import org.apache.hadoop.fs.s3a.S3ATestUtils;
|
import org.apache.hadoop.fs.s3a.S3ATestUtils;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
|
|
||||||
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
|
import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
|
||||||
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
|
|
||||||
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Extends FileContextCreateMkdirBaseTest for a S3a FileContext.
|
* Extends FileContextCreateMkdirBaseTest for a S3a FileContext.
|
||||||
@ -30,11 +28,9 @@ public class ITestS3AFileContextCreateMkdir
|
|||||||
|
|
||||||
@Before
|
@Before
|
||||||
public void setUp() throws Exception {
|
public void setUp() throws Exception {
|
||||||
Configuration conf = new Configuration();
|
Configuration conf = setPerformanceFlags(
|
||||||
removeBaseAndBucketOverrides(
|
new Configuration(),
|
||||||
conf,
|
null);
|
||||||
FS_S3A_CREATE_PERFORMANCE,
|
|
||||||
FS_S3A_PERFORMANCE_FLAGS);
|
|
||||||
fc = S3ATestUtils.createTestFileContext(conf);
|
fc = S3ATestUtils.createTestFileContext(conf);
|
||||||
super.setUp();
|
super.setUp();
|
||||||
}
|
}
|
||||||
|
@ -20,9 +20,7 @@
|
|||||||
import org.apache.hadoop.fs.FileContextCreateMkdirBaseTest;
|
import org.apache.hadoop.fs.FileContextCreateMkdirBaseTest;
|
||||||
import org.apache.hadoop.fs.s3a.S3ATestUtils;
|
import org.apache.hadoop.fs.s3a.S3ATestUtils;
|
||||||
|
|
||||||
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
|
import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
|
||||||
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
|
|
||||||
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
|
|
||||||
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
|
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -34,12 +32,8 @@ public class ITestS3AFileContextCreateMkdirCreatePerf
|
|||||||
|
|
||||||
@Before
|
@Before
|
||||||
public void setUp() throws Exception {
|
public void setUp() throws Exception {
|
||||||
Configuration conf = new Configuration();
|
Configuration conf = setPerformanceFlags(
|
||||||
removeBaseAndBucketOverrides(
|
new Configuration(),
|
||||||
conf,
|
|
||||||
FS_S3A_CREATE_PERFORMANCE,
|
|
||||||
FS_S3A_PERFORMANCE_FLAGS);
|
|
||||||
conf.setStrings(FS_S3A_PERFORMANCE_FLAGS,
|
|
||||||
"mkdir");
|
"mkdir");
|
||||||
fc = S3ATestUtils.createTestFileContext(conf);
|
fc = S3ATestUtils.createTestFileContext(conf);
|
||||||
super.setUp();
|
super.setUp();
|
||||||
|
@ -27,6 +27,8 @@
|
|||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.fs.s3a.S3ATestUtils;
|
import org.apache.hadoop.fs.s3a.S3ATestUtils;
|
||||||
|
|
||||||
|
import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* S3A implementation of FileContextMainOperationsBaseTest.
|
* S3A implementation of FileContextMainOperationsBaseTest.
|
||||||
*/
|
*/
|
||||||
@ -36,7 +38,10 @@ public class ITestS3AFileContextMainOperations
|
|||||||
|
|
||||||
@Before
|
@Before
|
||||||
public void setUp() throws IOException, Exception {
|
public void setUp() throws IOException, Exception {
|
||||||
Configuration conf = new Configuration();
|
Configuration conf = setPerformanceFlags(
|
||||||
|
new Configuration(),
|
||||||
|
"");
|
||||||
|
|
||||||
fc = S3ATestUtils.createTestFileContext(conf);
|
fc = S3ATestUtils.createTestFileContext(conf);
|
||||||
super.setUp();
|
super.setUp();
|
||||||
}
|
}
|
||||||
|
@ -21,6 +21,8 @@
|
|||||||
import org.junit.Ignore;
|
import org.junit.Ignore;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* S3a implementation of FileContextURIBase.
|
* S3a implementation of FileContextURIBase.
|
||||||
*/
|
*/
|
||||||
@ -30,7 +32,9 @@ public class ITestS3AFileContextURI extends FileContextURIBase {
|
|||||||
|
|
||||||
@Before
|
@Before
|
||||||
public void setUp() throws IOException, Exception {
|
public void setUp() throws IOException, Exception {
|
||||||
conf = new Configuration();
|
conf = setPerformanceFlags(
|
||||||
|
new Configuration(),
|
||||||
|
"");
|
||||||
|
|
||||||
fc1 = S3ATestUtils.createTestFileContext(conf);
|
fc1 = S3ATestUtils.createTestFileContext(conf);
|
||||||
fc2 = S3ATestUtils.createTestFileContext(conf); //different object, same FS
|
fc2 = S3ATestUtils.createTestFileContext(conf); //different object, same FS
|
||||||
|
@ -42,9 +42,8 @@
|
|||||||
import static org.apache.hadoop.fs.contract.ContractTestUtils.toChar;
|
import static org.apache.hadoop.fs.contract.ContractTestUtils.toChar;
|
||||||
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_HEADER;
|
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_HEADER;
|
||||||
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
|
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
|
||||||
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
|
|
||||||
import static org.apache.hadoop.fs.s3a.Constants.XA_HEADER_PREFIX;
|
import static org.apache.hadoop.fs.s3a.Constants.XA_HEADER_PREFIX;
|
||||||
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
|
import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
|
||||||
import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_BULK_DELETE_REQUEST;
|
import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_BULK_DELETE_REQUEST;
|
||||||
import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_DELETE_REQUEST;
|
import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_DELETE_REQUEST;
|
||||||
import static org.apache.hadoop.fs.s3a.performance.OperationCost.CREATE_FILE_NO_OVERWRITE;
|
import static org.apache.hadoop.fs.s3a.performance.OperationCost.CREATE_FILE_NO_OVERWRITE;
|
||||||
@ -105,11 +104,9 @@ private OperationCost expected(OperationCost source) {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Configuration createConfiguration() {
|
public Configuration createConfiguration() {
|
||||||
final Configuration conf = super.createConfiguration();
|
final Configuration conf = setPerformanceFlags(
|
||||||
removeBaseAndBucketOverrides(conf,
|
super.createConfiguration(),
|
||||||
FS_S3A_CREATE_PERFORMANCE,
|
createPerformance ? "create" : "");
|
||||||
FS_S3A_PERFORMANCE_FLAGS);
|
|
||||||
conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, createPerformance);
|
|
||||||
S3ATestUtils.disableFilesystemCaching(conf);
|
S3ATestUtils.disableFilesystemCaching(conf);
|
||||||
return conf;
|
return conf;
|
||||||
}
|
}
|
||||||
|
@ -39,9 +39,7 @@
|
|||||||
import org.apache.hadoop.fs.s3a.Tristate;
|
import org.apache.hadoop.fs.s3a.Tristate;
|
||||||
import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum;
|
import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum;
|
||||||
|
|
||||||
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
|
import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
|
||||||
import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_PERFORMANCE_FLAGS;
|
|
||||||
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
|
|
||||||
import static org.apache.hadoop.fs.s3a.Statistic.*;
|
import static org.apache.hadoop.fs.s3a.Statistic.*;
|
||||||
import static org.apache.hadoop.fs.s3a.performance.OperationCost.*;
|
import static org.apache.hadoop.fs.s3a.performance.OperationCost.*;
|
||||||
import static org.apache.hadoop.fs.s3a.performance.OperationCostValidator.probe;
|
import static org.apache.hadoop.fs.s3a.performance.OperationCostValidator.probe;
|
||||||
@ -80,12 +78,9 @@ public ITestS3ADeleteCost(final String name,
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Configuration createConfiguration() {
|
public Configuration createConfiguration() {
|
||||||
Configuration conf = super.createConfiguration();
|
return setPerformanceFlags(
|
||||||
removeBaseAndBucketOverrides(conf,
|
super.createConfiguration(),
|
||||||
FS_S3A_CREATE_PERFORMANCE,
|
"");
|
||||||
FS_S3A_PERFORMANCE_FLAGS);
|
|
||||||
conf.setBoolean(FS_S3A_CREATE_PERFORMANCE, false);
|
|
||||||
return conf;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
Loading…
Reference in New Issue
Block a user