From e1c5ddf2aa854951142e234462978245cdb99e1d Mon Sep 17 00:00:00 2001 From: Ben Roling Date: Wed, 24 Apr 2019 14:55:58 +0100 Subject: [PATCH] HADOOP-16252. Add prefix to dynamo tables in tests. Contributed by Ben Roling. --- .../site/markdown/tools/hadoop-aws/testing.md | 6 +++++ .../hadoop/fs/s3a/AbstractS3ATestBase.java | 5 ++++ .../hadoop/fs/s3a/S3ATestConstants.java | 4 ++++ .../apache/hadoop/fs/s3a/S3ATestUtils.java | 10 ++++++++ .../s3guard/AbstractS3GuardToolTestBase.java | 6 +++-- .../s3guard/ITestDynamoDBMetadataStore.java | 24 ++++++++++++------- .../s3guard/ITestS3GuardConcurrentOps.java | 4 +++- .../s3a/s3guard/ITestS3GuardToolDynamoDB.java | 11 +++++---- 8 files changed, 55 insertions(+), 15 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md index 37a53020ee..707694c8c6 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md @@ -1074,6 +1074,12 @@ If the `s3guard` profile *is* set, property should be configured, and the name of that table should be different than what is used for fs.s3a.s3guard.ddb.table. The test table is destroyed and modified multiple times during the test. + 1. Several of the tests create and destroy DynamoDB tables. The table names + are prefixed with the value defined by + `fs.s3a.s3guard.test.dynamo.table.prefix` (default="s3guard.test."). The user + executing the tests will need sufficient privilege to create and destroy such + tables. If the tests abort uncleanly, these tables may be left behind, + incurring AWS charges. ### Scale Testing MetadataStore Directly diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java index 484d2dcfb3..c1942a9443 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java @@ -33,6 +33,7 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestDynamoTablePrefix; /** * An extension of the contract test base set up for S3A tests. @@ -129,6 +130,10 @@ protected void writeThenReadFile(Path path, int len) throws IOException { ContractTestUtils.verifyFileContents(getFileSystem(), path, data); } + protected String getTestTableName(String suffix) { + return getTestDynamoTablePrefix(getConfiguration()) + suffix; + } + /** * Assert that an exception failed with a specific status code. * @param e exception diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java index 5f28c3012e..9d6e1ce00b 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java @@ -158,6 +158,10 @@ public interface S3ATestConstants { String TEST_S3GUARD_IMPLEMENTATION_DYNAMO = "dynamo"; String TEST_S3GUARD_IMPLEMENTATION_NONE = "none"; + String TEST_S3GUARD_DYNAMO_TABLE_PREFIX = + "fs.s3a.s3guard.test.dynamo.table.prefix"; + String TEST_S3GUARD_DYNAMO_TABLE_PREFIX_DEFAULT = "s3guard.test."; + /** * Timeout in Milliseconds for standard tests: {@value}. */ diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java index 5068524bf0..260ecdd71d 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java @@ -708,6 +708,16 @@ public static String getTestBucketName(final Configuration conf) { return URI.create(bucket).getHost(); } + /** + * Get the prefix for DynamoDB table names used in tests. + * @param conf configuration to scan. + * @return the table name prefix + */ + public static String getTestDynamoTablePrefix(final Configuration conf) { + return getTestProperty(conf, TEST_S3GUARD_DYNAMO_TABLE_PREFIX, + TEST_S3GUARD_DYNAMO_TABLE_PREFIX_DEFAULT); + } + /** * Remove any values from a bucket. * @param bucket bucket whose overrides are to be removed. Can be null/empty diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java index d4e4122eae..ad4691a6d9 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java @@ -59,6 +59,7 @@ import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_NAME_KEY; import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_METASTORE_NULL; import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestDynamoTablePrefix; import static org.apache.hadoop.fs.s3a.S3AUtils.clearBucketOption; import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.E_BAD_STATE; import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.SUCCESS; @@ -71,7 +72,7 @@ public abstract class AbstractS3GuardToolTestBase extends AbstractS3ATestBase { protected static final String OWNER = "hdfs"; - protected static final String DYNAMODB_TABLE = "dynamodb://ireland-team"; + protected static final String DYNAMODB_TABLE = "ireland-team"; protected static final String S3A_THIS_BUCKET_DOES_NOT_EXIST = "s3a://this-bucket-does-not-exist-00000000000"; @@ -520,7 +521,8 @@ public void testInitFailsIfNoBucketNameOrDDBTableSet() throws Exception { ByteArrayOutputStream buf = new ByteArrayOutputStream(); S3GuardTool.Diff cmd = new S3GuardTool.Diff(fs.getConf()); cmd.setStore(ms); - exec(0, "", cmd, buf, "diff", "-meta", DYNAMODB_TABLE, testPath.toString()); + String table = "dynamo://" + getTestTableName(DYNAMODB_TABLE); + exec(0, "", cmd, buf, "diff", "-meta", table, testPath.toString()); Set actualOnS3 = new HashSet<>(); Set actualOnMS = new HashSet<>(); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestDynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestDynamoDBMetadataStore.java index 2f8146dfd3..158f13d3fc 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestDynamoDBMetadataStore.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestDynamoDBMetadataStore.java @@ -269,7 +269,8 @@ private S3AFileSystem getFileSystem() { @Test public void testInitialize() throws IOException { final S3AFileSystem s3afs = this.fileSystem; - final String tableName = "testInitialize"; + final String tableName = + getTestTableName("testInitialize"); final Configuration conf = s3afs.getConf(); conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName); try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) { @@ -293,7 +294,8 @@ public void testInitialize() throws IOException { */ @Test public void testInitializeWithConfiguration() throws IOException { - final String tableName = "testInitializeWithConfiguration"; + final String tableName = + getTestTableName("testInitializeWithConfiguration"); final Configuration conf = getFileSystem().getConf(); conf.unset(S3GUARD_DDB_TABLE_NAME_KEY); String savedRegion = conf.get(S3GUARD_DDB_REGION_KEY, @@ -430,7 +432,7 @@ public void testItemLacksVersion() throws Throwable { */ @Test public void testTableVersionRequired() throws Exception { - String tableName = "testTableVersionRequired"; + String tableName = getTestTableName("testTableVersionRequired"); Configuration conf = getFileSystem().getConf(); int maxRetries = conf.getInt(S3GUARD_DDB_MAX_RETRIES, S3GUARD_DDB_MAX_RETRIES_DEFAULT); @@ -457,7 +459,7 @@ public void testTableVersionRequired() throws Exception { */ @Test public void testTableVersionMismatch() throws Exception { - String tableName = "testTableVersionMismatch"; + String tableName = getTestTableName("testTableVersionMismatch"); Configuration conf = getFileSystem().getConf(); conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName); @@ -484,7 +486,8 @@ public void testTableVersionMismatch() throws Exception { */ @Test public void testFailNonexistentTable() throws IOException { - final String tableName = "testFailNonexistentTable"; + final String tableName = + getTestTableName("testFailNonexistentTable"); final S3AFileSystem s3afs = getFileSystem(); final Configuration conf = s3afs.getConf(); conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName); @@ -600,7 +603,8 @@ public void testMovePopulatesAncestors() throws IOException { @Test public void testProvisionTable() throws Exception { - final String tableName = "testProvisionTable-" + UUID.randomUUID(); + final String tableName + = getTestTableName("testProvisionTable-" + UUID.randomUUID()); Configuration conf = getFileSystem().getConf(); conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName); @@ -631,7 +635,7 @@ public void testProvisionTable() throws Exception { @Test public void testDeleteTable() throws Exception { - final String tableName = "testDeleteTable"; + final String tableName = getTestTableName("testDeleteTable"); Path testPath = new Path(new Path(fsUri), "/" + tableName); final S3AFileSystem s3afs = getFileSystem(); final Configuration conf = s3afs.getConf(); @@ -666,7 +670,8 @@ public void testTableTagging() throws IOException { propKey -> conf.unset(S3GUARD_DDB_TABLE_TAG + propKey) ); - String tableName = "testTableTagging-" + UUID.randomUUID(); + String tableName = + getTestTableName("testTableTagging-" + UUID.randomUUID()); conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName); conf.set(S3GUARD_DDB_TABLE_CREATE_KEY, "true"); @@ -765,4 +770,7 @@ private void verifyTableNotExist(String tableName, DynamoDB dynamoDB) throws () -> dynamoDB.getTable(tableName).describe()); } + private String getTestTableName(String suffix) { + return getTestDynamoTablePrefix(s3AContract.getConf()) + suffix; + } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardConcurrentOps.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardConcurrentOps.java index 22a1efd2a4..fa7d1dcf46 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardConcurrentOps.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardConcurrentOps.java @@ -106,7 +106,9 @@ public void testConcurrentTableCreations() throws Exception { try { DynamoDB db = ms.getDynamoDB(); - String tableName = "testConcurrentTableCreations" + new Random().nextInt(); + String tableName = + getTestTableName("testConcurrentTableCreations" + + new Random().nextInt()); conf.setBoolean(Constants.S3GUARD_DDB_TABLE_CREATE_KEY, true); conf.set(Constants.S3GUARD_DDB_TABLE_NAME_KEY, tableName); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java index aaaae2e462..98c1e998ed 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java @@ -44,6 +44,7 @@ import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_REGION_KEY; import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_NAME_KEY; import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_TAG; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestDynamoTablePrefix; import static org.apache.hadoop.fs.s3a.S3AUtils.setBucketOption; import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.*; import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.*; @@ -80,7 +81,8 @@ private static boolean exist(DynamoDB dynamoDB, String tableName) { @Test public void testInvalidRegion() throws Exception { - final String testTableName = "testInvalidRegion" + new Random().nextInt(); + final String testTableName = + getTestTableName("testInvalidRegion" + new Random().nextInt()); final String testRegion = "invalidRegion"; // Initialize MetadataStore final Init initCmd = new Init(getFileSystem().getConf()); @@ -117,7 +119,7 @@ public void testDynamoTableTagging() throws Exception { ); conf.set(S3GUARD_DDB_TABLE_NAME_KEY, - "testDynamoTableTagging-" + UUID.randomUUID()); + getTestTableName("testDynamoTableTagging-" + UUID.randomUUID())); S3GuardTool.Init cmdR = new S3GuardTool.Init(conf); Map tagMap = new HashMap<>(); tagMap.put("hello", "dynamo"); @@ -165,7 +167,8 @@ private DDBCapacities getCapacities() throws IOException { @Test public void testDynamoDBInitDestroyCycle() throws Throwable { - String testTableName = "testDynamoDBInitDestroy" + new Random().nextInt(); + String testTableName = + getTestTableName("testDynamoDBInitDestroy" + new Random().nextInt()); String testS3Url = path(testTableName).toString(); S3AFileSystem fs = getFileSystem(); DynamoDB db = null; @@ -284,7 +287,7 @@ private S3GuardTool newSetCapacity() { public void testDestroyUnknownTable() throws Throwable { run(S3GuardTool.Destroy.NAME, "-region", "us-west-2", - "-meta", DYNAMODB_TABLE); + "-meta", "dynamodb://" + getTestTableName(DYNAMODB_TABLE)); } }