diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md index cb07a49e6d..fb141ddecc 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md @@ -996,11 +996,7 @@ When the `s3guard` profile is enabled, following profiles can be specified: * `dynamo`: use an AWS-hosted DynamoDB table; creating the table if it does not exist. You will have to pay the bills for DynamoDB web service. -* `dynamodblocal`: use an in-memory DynamoDBLocal server instead of real AWS - DynamoDB web service; launch the server and creating the table. - You won't be charged bills for using DynamoDB in test. As it runs in-JVM, - the table isn't shared across other tests running in parallel. -* `non-auth`: treat the S3Guard metadata as authoritative. +* `auth`: treat the S3Guard metadata as authoritative. ```bash mvn -T 1C verify -Dparallel-tests -DtestsThreadCount=6 -Ds3guard -Ddynamo -Dauth @@ -1022,6 +1018,10 @@ If the `s3guard` profile *is* set, 1. The S3Guard options from maven (the dynamo and authoritative flags) overwrite any previously set in the configuration files. 1. DynamoDB will be configured to create any missing tables. +1. When using DynamoDB and running ITestDynamoDBMetadataStore, the fs.s3a.s3guard.ddb.test.table +property should be configured, and the name of that table should be different + than what is used for fs.s3a.s3guard.ddb.table. The test table is destroyed + and modified multiple times during the test. ### Scale Testing MetadataStore Directly diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestDynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestDynamoDBMetadataStore.java index 5ae8356f1a..e958919d54 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestDynamoDBMetadataStore.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestDynamoDBMetadataStore.java @@ -116,7 +116,7 @@ protected Path path(String filepath) { @Override public void setUp() throws Exception { Configuration conf = prepareTestConfiguration(new Configuration()); - assertThatDynamoMetadataStoreImpl(conf); + assumeThatDynamoMetadataStoreImpl(conf); Assume.assumeTrue("Test DynamoDB table name should be set to run " + "integration tests.", testDynamoDBTableName != null); conf.set(S3GUARD_DDB_TABLE_NAME_KEY, testDynamoDBTableName); @@ -144,10 +144,24 @@ public void setUp() throws Exception { @BeforeClass public static void beforeClassSetup() throws IOException { Configuration conf = prepareTestConfiguration(new Configuration()); - assertThatDynamoMetadataStoreImpl(conf); + assumeThatDynamoMetadataStoreImpl(conf); testDynamoDBTableName = conf.get(S3GUARD_DDB_TEST_TABLE_NAME_KEY); - Assume.assumeTrue("Test DynamoDB table name should be set to run " + + // We should assert that the table name is configured, so the test should + // fail if it's not configured. + assertTrue("Test DynamoDB table name '" + + S3GUARD_DDB_TEST_TABLE_NAME_KEY + "' should be set to run " + "integration tests.", testDynamoDBTableName != null); + + // We should assert that the test table is not the same as the production + // table, as the test table could be modified and destroyed multiple + // times during the test. + assertTrue("Test DynamoDB table name: '" + + S3GUARD_DDB_TEST_TABLE_NAME_KEY + "' and production table name: '" + + S3GUARD_DDB_TABLE_NAME_KEY + "' can not be the same.", + !conf.get(S3GUARD_DDB_TABLE_NAME_KEY).equals(testDynamoDBTableName)); + + // We can use that table in the test if these assertions are valid conf.set(S3GUARD_DDB_TABLE_NAME_KEY, testDynamoDBTableName); LOG.debug("Creating static ddbms which will be shared between tests."); @@ -169,7 +183,7 @@ public static void afterClassTeardown() { } } - private static void assertThatDynamoMetadataStoreImpl(Configuration conf){ + private static void assumeThatDynamoMetadataStoreImpl(Configuration conf){ Assume.assumeTrue("Test only applies when DynamoDB is used for S3Guard", conf.get(Constants.S3_METADATA_STORE_IMPL).equals( Constants.S3GUARD_METASTORE_DYNAMO));