HADOOP-15987. ITestDynamoDBMetadataStore should check if table configured properly. Contributed by Gabor Bota.
This commit is contained in:
parent
39dc7345b8
commit
c35de95a22
@ -996,11 +996,7 @@ When the `s3guard` profile is enabled, following profiles can be specified:
|
|||||||
|
|
||||||
* `dynamo`: use an AWS-hosted DynamoDB table; creating the table if it does
|
* `dynamo`: use an AWS-hosted DynamoDB table; creating the table if it does
|
||||||
not exist. You will have to pay the bills for DynamoDB web service.
|
not exist. You will have to pay the bills for DynamoDB web service.
|
||||||
* `dynamodblocal`: use an in-memory DynamoDBLocal server instead of real AWS
|
* `auth`: treat the S3Guard metadata as authoritative.
|
||||||
DynamoDB web service; launch the server and creating the table.
|
|
||||||
You won't be charged bills for using DynamoDB in test. As it runs in-JVM,
|
|
||||||
the table isn't shared across other tests running in parallel.
|
|
||||||
* `non-auth`: treat the S3Guard metadata as authoritative.
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
mvn -T 1C verify -Dparallel-tests -DtestsThreadCount=6 -Ds3guard -Ddynamo -Dauth
|
mvn -T 1C verify -Dparallel-tests -DtestsThreadCount=6 -Ds3guard -Ddynamo -Dauth
|
||||||
@ -1022,6 +1018,10 @@ If the `s3guard` profile *is* set,
|
|||||||
1. The S3Guard options from maven (the dynamo and authoritative flags)
|
1. The S3Guard options from maven (the dynamo and authoritative flags)
|
||||||
overwrite any previously set in the configuration files.
|
overwrite any previously set in the configuration files.
|
||||||
1. DynamoDB will be configured to create any missing tables.
|
1. DynamoDB will be configured to create any missing tables.
|
||||||
|
1. When using DynamoDB and running ITestDynamoDBMetadataStore, the fs.s3a.s3guard.ddb.test.table
|
||||||
|
property should be configured, and the name of that table should be different
|
||||||
|
than what is used for fs.s3a.s3guard.ddb.table. The test table is destroyed
|
||||||
|
and modified multiple times during the test.
|
||||||
|
|
||||||
|
|
||||||
### Scale Testing MetadataStore Directly
|
### Scale Testing MetadataStore Directly
|
||||||
|
@ -116,7 +116,7 @@ protected Path path(String filepath) {
|
|||||||
@Override
|
@Override
|
||||||
public void setUp() throws Exception {
|
public void setUp() throws Exception {
|
||||||
Configuration conf = prepareTestConfiguration(new Configuration());
|
Configuration conf = prepareTestConfiguration(new Configuration());
|
||||||
assertThatDynamoMetadataStoreImpl(conf);
|
assumeThatDynamoMetadataStoreImpl(conf);
|
||||||
Assume.assumeTrue("Test DynamoDB table name should be set to run "
|
Assume.assumeTrue("Test DynamoDB table name should be set to run "
|
||||||
+ "integration tests.", testDynamoDBTableName != null);
|
+ "integration tests.", testDynamoDBTableName != null);
|
||||||
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, testDynamoDBTableName);
|
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, testDynamoDBTableName);
|
||||||
@ -144,10 +144,24 @@ public void setUp() throws Exception {
|
|||||||
@BeforeClass
|
@BeforeClass
|
||||||
public static void beforeClassSetup() throws IOException {
|
public static void beforeClassSetup() throws IOException {
|
||||||
Configuration conf = prepareTestConfiguration(new Configuration());
|
Configuration conf = prepareTestConfiguration(new Configuration());
|
||||||
assertThatDynamoMetadataStoreImpl(conf);
|
assumeThatDynamoMetadataStoreImpl(conf);
|
||||||
testDynamoDBTableName = conf.get(S3GUARD_DDB_TEST_TABLE_NAME_KEY);
|
testDynamoDBTableName = conf.get(S3GUARD_DDB_TEST_TABLE_NAME_KEY);
|
||||||
Assume.assumeTrue("Test DynamoDB table name should be set to run "
|
|
||||||
|
// We should assert that the table name is configured, so the test should
|
||||||
|
// fail if it's not configured.
|
||||||
|
assertTrue("Test DynamoDB table name '"
|
||||||
|
+ S3GUARD_DDB_TEST_TABLE_NAME_KEY + "' should be set to run "
|
||||||
+ "integration tests.", testDynamoDBTableName != null);
|
+ "integration tests.", testDynamoDBTableName != null);
|
||||||
|
|
||||||
|
// We should assert that the test table is not the same as the production
|
||||||
|
// table, as the test table could be modified and destroyed multiple
|
||||||
|
// times during the test.
|
||||||
|
assertTrue("Test DynamoDB table name: '"
|
||||||
|
+ S3GUARD_DDB_TEST_TABLE_NAME_KEY + "' and production table name: '"
|
||||||
|
+ S3GUARD_DDB_TABLE_NAME_KEY + "' can not be the same.",
|
||||||
|
!conf.get(S3GUARD_DDB_TABLE_NAME_KEY).equals(testDynamoDBTableName));
|
||||||
|
|
||||||
|
// We can use that table in the test if these assertions are valid
|
||||||
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, testDynamoDBTableName);
|
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, testDynamoDBTableName);
|
||||||
|
|
||||||
LOG.debug("Creating static ddbms which will be shared between tests.");
|
LOG.debug("Creating static ddbms which will be shared between tests.");
|
||||||
@ -169,7 +183,7 @@ public static void afterClassTeardown() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void assertThatDynamoMetadataStoreImpl(Configuration conf){
|
private static void assumeThatDynamoMetadataStoreImpl(Configuration conf){
|
||||||
Assume.assumeTrue("Test only applies when DynamoDB is used for S3Guard",
|
Assume.assumeTrue("Test only applies when DynamoDB is used for S3Guard",
|
||||||
conf.get(Constants.S3_METADATA_STORE_IMPL).equals(
|
conf.get(Constants.S3_METADATA_STORE_IMPL).equals(
|
||||||
Constants.S3GUARD_METASTORE_DYNAMO));
|
Constants.S3GUARD_METASTORE_DYNAMO));
|
||||||
|
Loading…
Reference in New Issue
Block a user