HADOOP-15987. ITestDynamoDBMetadataStore should check if table configured properly. Contributed by Gabor Bota.

This commit is contained in:
Sean Mackrory 2018-12-10 10:46:46 -07:00
parent 39dc7345b8
commit c35de95a22
2 changed files with 23 additions and 9 deletions

View File

@ -996,11 +996,7 @@ When the `s3guard` profile is enabled, following profiles can be specified:
* `dynamo`: use an AWS-hosted DynamoDB table; creating the table if it does * `dynamo`: use an AWS-hosted DynamoDB table; creating the table if it does
not exist. You will have to pay the bills for DynamoDB web service. not exist. You will have to pay the bills for DynamoDB web service.
* `dynamodblocal`: use an in-memory DynamoDBLocal server instead of real AWS * `auth`: treat the S3Guard metadata as authoritative.
DynamoDB web service; launch the server and creating the table.
You won't be charged bills for using DynamoDB in test. As it runs in-JVM,
the table isn't shared across other tests running in parallel.
* `non-auth`: treat the S3Guard metadata as authoritative.
```bash ```bash
mvn -T 1C verify -Dparallel-tests -DtestsThreadCount=6 -Ds3guard -Ddynamo -Dauth mvn -T 1C verify -Dparallel-tests -DtestsThreadCount=6 -Ds3guard -Ddynamo -Dauth
@ -1022,6 +1018,10 @@ If the `s3guard` profile *is* set,
1. The S3Guard options from maven (the dynamo and authoritative flags) 1. The S3Guard options from maven (the dynamo and authoritative flags)
overwrite any previously set in the configuration files. overwrite any previously set in the configuration files.
1. DynamoDB will be configured to create any missing tables. 1. DynamoDB will be configured to create any missing tables.
1. When using DynamoDB and running ITestDynamoDBMetadataStore, the fs.s3a.s3guard.ddb.test.table
property should be configured, and the name of that table should be different
than what is used for fs.s3a.s3guard.ddb.table. The test table is destroyed
and modified multiple times during the test.
### Scale Testing MetadataStore Directly ### Scale Testing MetadataStore Directly

View File

@ -116,7 +116,7 @@ protected Path path(String filepath) {
@Override @Override
public void setUp() throws Exception { public void setUp() throws Exception {
Configuration conf = prepareTestConfiguration(new Configuration()); Configuration conf = prepareTestConfiguration(new Configuration());
assertThatDynamoMetadataStoreImpl(conf); assumeThatDynamoMetadataStoreImpl(conf);
Assume.assumeTrue("Test DynamoDB table name should be set to run " Assume.assumeTrue("Test DynamoDB table name should be set to run "
+ "integration tests.", testDynamoDBTableName != null); + "integration tests.", testDynamoDBTableName != null);
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, testDynamoDBTableName); conf.set(S3GUARD_DDB_TABLE_NAME_KEY, testDynamoDBTableName);
@ -144,10 +144,24 @@ public void setUp() throws Exception {
@BeforeClass @BeforeClass
public static void beforeClassSetup() throws IOException { public static void beforeClassSetup() throws IOException {
Configuration conf = prepareTestConfiguration(new Configuration()); Configuration conf = prepareTestConfiguration(new Configuration());
assertThatDynamoMetadataStoreImpl(conf); assumeThatDynamoMetadataStoreImpl(conf);
testDynamoDBTableName = conf.get(S3GUARD_DDB_TEST_TABLE_NAME_KEY); testDynamoDBTableName = conf.get(S3GUARD_DDB_TEST_TABLE_NAME_KEY);
Assume.assumeTrue("Test DynamoDB table name should be set to run "
// We should assert that the table name is configured, so the test should
// fail if it's not configured.
assertTrue("Test DynamoDB table name '"
+ S3GUARD_DDB_TEST_TABLE_NAME_KEY + "' should be set to run "
+ "integration tests.", testDynamoDBTableName != null); + "integration tests.", testDynamoDBTableName != null);
// We should assert that the test table is not the same as the production
// table, as the test table could be modified and destroyed multiple
// times during the test.
assertTrue("Test DynamoDB table name: '"
+ S3GUARD_DDB_TEST_TABLE_NAME_KEY + "' and production table name: '"
+ S3GUARD_DDB_TABLE_NAME_KEY + "' can not be the same.",
!conf.get(S3GUARD_DDB_TABLE_NAME_KEY).equals(testDynamoDBTableName));
// We can use that table in the test if these assertions are valid
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, testDynamoDBTableName); conf.set(S3GUARD_DDB_TABLE_NAME_KEY, testDynamoDBTableName);
LOG.debug("Creating static ddbms which will be shared between tests."); LOG.debug("Creating static ddbms which will be shared between tests.");
@ -169,7 +183,7 @@ public static void afterClassTeardown() {
} }
} }
private static void assertThatDynamoMetadataStoreImpl(Configuration conf){ private static void assumeThatDynamoMetadataStoreImpl(Configuration conf){
Assume.assumeTrue("Test only applies when DynamoDB is used for S3Guard", Assume.assumeTrue("Test only applies when DynamoDB is used for S3Guard",
conf.get(Constants.S3_METADATA_STORE_IMPL).equals( conf.get(Constants.S3_METADATA_STORE_IMPL).equals(
Constants.S3GUARD_METASTORE_DYNAMO)); Constants.S3GUARD_METASTORE_DYNAMO));