big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-15987. ITestDynamoDBMetadataStore should check if table configured properly. Contributed by Gabor Bota.

Browse Source
This commit is contained in:
Sean Mackrory 2018-12-10 10:46:46 -07:00
parent 39dc7345b8
commit c35de95a22
2 changed files with 23 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
View File

@ -996,11 +996,7 @@ When the `s3guard` profile is enabled, following profiles can be specified:
* `dynamo`: use an AWS-hosted DynamoDB table; creating the table if it does
not exist. You will have to pay the bills for DynamoDB web service.
* `dynamodblocal`: use an in-memory DynamoDBLocal server instead of real AWS
DynamoDB web service; launch the server and creating the table.
You won't be charged bills for using DynamoDB in test. As it runs in-JVM,
the table isn't shared across other tests running in parallel.
* `non-auth`: treat the S3Guard metadata as authoritative.
* `auth`: treat the S3Guard metadata as authoritative.
```bash
mvn -T 1C verify -Dparallel-tests -DtestsThreadCount=6 -Ds3guard -Ddynamo -Dauth
@ -1022,6 +1018,10 @@ If the `s3guard` profile *is* set,
1. The S3Guard options from maven (the dynamo and authoritative flags)
overwrite any previously set in the configuration files.
1. DynamoDB will be configured to create any missing tables.
1. When using DynamoDB and running ITestDynamoDBMetadataStore, the fs.s3a.s3guard.ddb.test.table
property should be configured, and the name of that table should be different
than what is used for fs.s3a.s3guard.ddb.table. The test table is destroyed
and modified multiple times during the test.
### Scale Testing MetadataStore Directly

22
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestDynamoDBMetadataStore.java
View File

@ -116,7 +116,7 @@ protected Path path(String filepath) {
@Override
public void setUp() throws Exception {
Configuration conf = prepareTestConfiguration(new Configuration());
assertThatDynamoMetadataStoreImpl(conf);
assumeThatDynamoMetadataStoreImpl(conf);
Assume.assumeTrue("Test DynamoDB table name should be set to run "
+ "integration tests.", testDynamoDBTableName != null);
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, testDynamoDBTableName);
@ -144,10 +144,24 @@ public void setUp() throws Exception {
@BeforeClass
public static void beforeClassSetup() throws IOException {
Configuration conf = prepareTestConfiguration(new Configuration());
assertThatDynamoMetadataStoreImpl(conf);
assumeThatDynamoMetadataStoreImpl(conf);
testDynamoDBTableName = conf.get(S3GUARD_DDB_TEST_TABLE_NAME_KEY);
Assume.assumeTrue("Test DynamoDB table name should be set to run "
// We should assert that the table name is configured, so the test should
// fail if it's not configured.
assertTrue("Test DynamoDB table name '"
+ S3GUARD_DDB_TEST_TABLE_NAME_KEY + "' should be set to run "
+ "integration tests.", testDynamoDBTableName != null);
// We should assert that the test table is not the same as the production
// table, as the test table could be modified and destroyed multiple
// times during the test.
assertTrue("Test DynamoDB table name: '"
+ S3GUARD_DDB_TEST_TABLE_NAME_KEY + "' and production table name: '"
+ S3GUARD_DDB_TABLE_NAME_KEY + "' can not be the same.",
!conf.get(S3GUARD_DDB_TABLE_NAME_KEY).equals(testDynamoDBTableName));
// We can use that table in the test if these assertions are valid
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, testDynamoDBTableName);
LOG.debug("Creating static ddbms which will be shared between tests.");
@ -169,7 +183,7 @@ public static void afterClassTeardown() {
}
}
private static void assertThatDynamoMetadataStoreImpl(Configuration conf){
private static void assumeThatDynamoMetadataStoreImpl(Configuration conf){
Assume.assumeTrue("Test only applies when DynamoDB is used for S3Guard",
conf.get(Constants.S3_METADATA_STORE_IMPL).equals(
Constants.S3GUARD_METASTORE_DYNAMO));