HADOOP-15636. Follow-up from HADOOP-14918; restoring test under new name. Contributed by Gabor Bota.
This commit is contained in:
Sean Mackrory
parent
ed9d60e888
commit
59adeb8d7f
@ -0,0 +1,649 @@
|
|||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.fs.s3a.s3guard;
|
||||||
|
|
||||||
|
import java.io.FileNotFoundException;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.URI;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
|
||||||
|
import com.amazonaws.services.dynamodbv2.document.Item;
|
||||||
|
import com.amazonaws.services.dynamodbv2.document.PrimaryKey;
|
||||||
|
import com.amazonaws.services.dynamodbv2.document.Table;
|
||||||
|
import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughputDescription;
|
||||||
|
import com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException;
|
||||||
|
import com.amazonaws.services.dynamodbv2.model.TableDescription;
|
||||||
|
|
||||||
|
import com.google.common.collect.Lists;
|
||||||
|
import org.apache.commons.collections.CollectionUtils;
|
||||||
|
import org.apache.hadoop.fs.contract.s3a.S3AContract;
|
||||||
|
import org.apache.hadoop.fs.s3a.Constants;
|
||||||
|
import org.apache.hadoop.fs.s3a.Tristate;
|
||||||
|
|
||||||
|
import org.apache.hadoop.io.IOUtils;
|
||||||
|
import org.junit.AfterClass;
|
||||||
|
import org.junit.Assume;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.fs.FileStatus;
|
||||||
|
import org.apache.hadoop.fs.Path;
|
||||||
|
import org.apache.hadoop.fs.s3a.S3AFileStatus;
|
||||||
|
import org.apache.hadoop.fs.s3a.S3AFileSystem;
|
||||||
|
import org.apache.hadoop.security.UserGroupInformation;
|
||||||
|
|
||||||
|
import static org.apache.hadoop.fs.s3a.Constants.*;
|
||||||
|
import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
|
||||||
|
import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.*;
|
||||||
|
import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.*;
|
||||||
|
import static org.apache.hadoop.test.LambdaTestUtils.*;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test that {@link DynamoDBMetadataStore} implements {@link MetadataStore}.
|
||||||
|
*
|
||||||
|
* In this integration test, we use a real AWS DynamoDB. A
|
||||||
|
* {@link DynamoDBMetadataStore} object is created in the @BeforeClass method,
|
||||||
|
* and shared for all test in the @BeforeClass method. You will be charged
|
||||||
|
* bills for AWS S3 or DynamoDB when you run these tests.
|
||||||
|
*
|
||||||
|
* According to the base class, every test case will have independent contract
|
||||||
|
* to create a new {@link S3AFileSystem} instance and initializes it.
|
||||||
|
* A table will be created and shared between the tests,
|
||||||
|
*/
|
||||||
|
public class ITestDynamoDBMetadataStore extends MetadataStoreTestBase {
|
||||||
|
private static final Logger LOG =
|
||||||
|
LoggerFactory.getLogger(ITestDynamoDBMetadataStore.class);
|
||||||
|
public static final PrimaryKey
|
||||||
|
VERSION_MARKER_PRIMARY_KEY = createVersionMarkerPrimaryKey(
|
||||||
|
DynamoDBMetadataStore.VERSION_MARKER);
|
||||||
|
|
||||||
|
private S3AFileSystem fileSystem;
|
||||||
|
private S3AContract s3AContract;
|
||||||
|
|
||||||
|
private URI fsUri;
|
||||||
|
|
||||||
|
private String bucket;
|
||||||
|
|
||||||
|
private static DynamoDBMetadataStore ddbmsStatic;
|
||||||
|
|
||||||
|
private static String TEST_DYNAMODB_TABLE_NAME;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a path under the test path provided by
|
||||||
|
* the FS contract.
|
||||||
|
* @param filepath path string in
|
||||||
|
* @return a path qualified by the test filesystem
|
||||||
|
*/
|
||||||
|
protected Path path(String filepath) {
|
||||||
|
return getFileSystem().makeQualified(
|
||||||
|
new Path(s3AContract.getTestPath(), filepath));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setUp() throws Exception {
|
||||||
|
Configuration conf = prepareTestConfiguration(new Configuration());
|
||||||
|
assertThatDynamoMetadataStoreImpl(conf);
|
||||||
|
Assume.assumeTrue("Test DynamoDB table name should be set to run "
|
||||||
|
+ "integration tests.", TEST_DYNAMODB_TABLE_NAME != null);
|
||||||
|
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, TEST_DYNAMODB_TABLE_NAME);
|
||||||
|
|
||||||
|
s3AContract = new S3AContract(conf);
|
||||||
|
s3AContract.init();
|
||||||
|
|
||||||
|
fileSystem = (S3AFileSystem) s3AContract.getTestFileSystem();
|
||||||
|
assume("No test filesystem", s3AContract.isEnabled());
|
||||||
|
assertNotNull("No test filesystem", fileSystem);
|
||||||
|
fsUri = fileSystem.getUri();
|
||||||
|
bucket = fileSystem.getBucket();
|
||||||
|
|
||||||
|
try{
|
||||||
|
super.setUp();
|
||||||
|
} catch (FileNotFoundException e){
|
||||||
|
LOG.warn("MetadataStoreTestBase setup failed. Waiting for table to be "
|
||||||
|
+ "deleted before trying again.");
|
||||||
|
ddbmsStatic.getTable().waitForDelete();
|
||||||
|
super.setUp();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void beforeClassSetup() throws IOException {
|
||||||
|
Configuration conf = prepareTestConfiguration(new Configuration());
|
||||||
|
assertThatDynamoMetadataStoreImpl(conf);
|
||||||
|
TEST_DYNAMODB_TABLE_NAME = conf.get(S3GUARD_DDB_TEST_TABLE_NAME_KEY);
|
||||||
|
Assume.assumeTrue("Test DynamoDB table name should be set to run "
|
||||||
|
+ "integration tests.", TEST_DYNAMODB_TABLE_NAME != null);
|
||||||
|
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, TEST_DYNAMODB_TABLE_NAME);
|
||||||
|
|
||||||
|
LOG.debug("Creating static ddbms which will be shared between tests.");
|
||||||
|
ddbmsStatic = new DynamoDBMetadataStore();
|
||||||
|
ddbmsStatic.initialize(conf);
|
||||||
|
}
|
||||||
|
|
||||||
|
@AfterClass
|
||||||
|
public static void afterClassTeardown() {
|
||||||
|
LOG.debug("Destroying static DynamoDBMetadataStore.");
|
||||||
|
if (ddbmsStatic != null) {
|
||||||
|
try {
|
||||||
|
ddbmsStatic.destroy();
|
||||||
|
} catch (Exception e) {
|
||||||
|
LOG.warn("Failed to destroy tables in teardown", e);
|
||||||
|
}
|
||||||
|
IOUtils.closeStream(ddbmsStatic);
|
||||||
|
ddbmsStatic = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void assertThatDynamoMetadataStoreImpl(Configuration conf){
|
||||||
|
Assume.assumeTrue("Test only applies when DynamoDB is used for S3Guard",
|
||||||
|
conf.get(Constants.S3_METADATA_STORE_IMPL).equals(
|
||||||
|
Constants.S3GUARD_METASTORE_DYNAMO));
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void tearDown() throws Exception {
|
||||||
|
LOG.info("Removing data from ddbms table in teardown.");
|
||||||
|
// The following is a way to be sure the table will be cleared and there
|
||||||
|
// will be no leftovers after the test.
|
||||||
|
PathMetadata meta = ddbmsStatic.get(strToPath("/"));
|
||||||
|
if (meta != null){
|
||||||
|
for (DescendantsIterator desc = new DescendantsIterator(ddbmsStatic, meta);
|
||||||
|
desc.hasNext();) {
|
||||||
|
ddbmsStatic.forgetMetadata(desc.next().getPath());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fileSystem.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Each contract has its own S3AFileSystem and DynamoDBMetadataStore objects.
|
||||||
|
*/
|
||||||
|
private class DynamoDBMSContract extends AbstractMSContract {
|
||||||
|
|
||||||
|
DynamoDBMSContract(Configuration conf) {
|
||||||
|
}
|
||||||
|
|
||||||
|
DynamoDBMSContract() {
|
||||||
|
this(new Configuration());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public S3AFileSystem getFileSystem() {
|
||||||
|
return ITestDynamoDBMetadataStore.this.fileSystem;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DynamoDBMetadataStore getMetadataStore() {
|
||||||
|
return ITestDynamoDBMetadataStore.ddbmsStatic;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DynamoDBMSContract createContract() {
|
||||||
|
return new DynamoDBMSContract();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DynamoDBMSContract createContract(Configuration conf) {
|
||||||
|
return new DynamoDBMSContract(conf);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
FileStatus basicFileStatus(Path path, int size, boolean isDir)
|
||||||
|
throws IOException {
|
||||||
|
String owner = UserGroupInformation.getCurrentUser().getShortUserName();
|
||||||
|
return isDir
|
||||||
|
? new S3AFileStatus(true, path, owner)
|
||||||
|
: new S3AFileStatus(size, getModTime(), path, BLOCK_SIZE, owner);
|
||||||
|
}
|
||||||
|
|
||||||
|
private DynamoDBMetadataStore getDynamoMetadataStore() throws IOException {
|
||||||
|
return (DynamoDBMetadataStore) getContract().getMetadataStore();
|
||||||
|
}
|
||||||
|
|
||||||
|
private S3AFileSystem getFileSystem() {
|
||||||
|
return this.fileSystem;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This tests that after initialize() using an S3AFileSystem object, the
|
||||||
|
* instance should have been initialized successfully, and tables are ACTIVE.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void testInitialize() throws IOException {
|
||||||
|
final S3AFileSystem s3afs = this.fileSystem;
|
||||||
|
final String tableName = "testInitialize";
|
||||||
|
final Configuration conf = s3afs.getConf();
|
||||||
|
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName);
|
||||||
|
try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) {
|
||||||
|
ddbms.initialize(s3afs);
|
||||||
|
verifyTableInitialized(tableName, ddbms.getDynamoDB());
|
||||||
|
assertNotNull(ddbms.getTable());
|
||||||
|
assertEquals(tableName, ddbms.getTable().getTableName());
|
||||||
|
String expectedRegion = conf.get(S3GUARD_DDB_REGION_KEY,
|
||||||
|
s3afs.getBucketLocation(bucket));
|
||||||
|
assertEquals("DynamoDB table should be in configured region or the same" +
|
||||||
|
" region as S3 bucket",
|
||||||
|
expectedRegion,
|
||||||
|
ddbms.getRegion());
|
||||||
|
ddbms.destroy();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This tests that after initialize() using a Configuration object, the
|
||||||
|
* instance should have been initialized successfully, and tables are ACTIVE.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void testInitializeWithConfiguration() throws IOException {
|
||||||
|
final String tableName = "testInitializeWithConfiguration";
|
||||||
|
final Configuration conf = getFileSystem().getConf();
|
||||||
|
conf.unset(S3GUARD_DDB_TABLE_NAME_KEY);
|
||||||
|
String savedRegion = conf.get(S3GUARD_DDB_REGION_KEY,
|
||||||
|
getFileSystem().getBucketLocation());
|
||||||
|
conf.unset(S3GUARD_DDB_REGION_KEY);
|
||||||
|
try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) {
|
||||||
|
ddbms.initialize(conf);
|
||||||
|
fail("Should have failed because the table name is not set!");
|
||||||
|
} catch (IllegalArgumentException ignored) {
|
||||||
|
}
|
||||||
|
// config table name
|
||||||
|
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName);
|
||||||
|
try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) {
|
||||||
|
ddbms.initialize(conf);
|
||||||
|
fail("Should have failed because as the region is not set!");
|
||||||
|
} catch (IllegalArgumentException ignored) {
|
||||||
|
}
|
||||||
|
// config region
|
||||||
|
conf.set(S3GUARD_DDB_REGION_KEY, savedRegion);
|
||||||
|
try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) {
|
||||||
|
ddbms.initialize(conf);
|
||||||
|
verifyTableInitialized(tableName, ddbms.getDynamoDB());
|
||||||
|
assertNotNull(ddbms.getTable());
|
||||||
|
assertEquals(tableName, ddbms.getTable().getTableName());
|
||||||
|
assertEquals("Unexpected key schema found!",
|
||||||
|
keySchema(),
|
||||||
|
ddbms.getTable().describe().getKeySchema());
|
||||||
|
ddbms.destroy();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test that for a large batch write request, the limit is handled correctly.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void testBatchWrite() throws IOException {
|
||||||
|
final int[] numMetasToDeleteOrPut = {
|
||||||
|
-1, // null
|
||||||
|
0, // empty collection
|
||||||
|
1, // one path
|
||||||
|
S3GUARD_DDB_BATCH_WRITE_REQUEST_LIMIT, // exact limit of a batch request
|
||||||
|
S3GUARD_DDB_BATCH_WRITE_REQUEST_LIMIT + 1 // limit + 1
|
||||||
|
};
|
||||||
|
DynamoDBMetadataStore ms = getDynamoMetadataStore();
|
||||||
|
for (int numOldMetas : numMetasToDeleteOrPut) {
|
||||||
|
for (int numNewMetas : numMetasToDeleteOrPut) {
|
||||||
|
doTestBatchWrite(numOldMetas, numNewMetas, ms);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void doTestBatchWrite(int numDelete, int numPut,
|
||||||
|
DynamoDBMetadataStore ms) throws IOException {
|
||||||
|
Path path = new Path(
|
||||||
|
"/ITestDynamoDBMetadataStore_testBatchWrite_" + numDelete + '_'
|
||||||
|
+ numPut);
|
||||||
|
final Path root = fileSystem.makeQualified(path);
|
||||||
|
final Path oldDir = new Path(root, "oldDir");
|
||||||
|
final Path newDir = new Path(root, "newDir");
|
||||||
|
LOG.info("doTestBatchWrite: oldDir={}, newDir={}", oldDir, newDir);
|
||||||
|
|
||||||
|
ms.put(new PathMetadata(basicFileStatus(oldDir, 0, true)));
|
||||||
|
ms.put(new PathMetadata(basicFileStatus(newDir, 0, true)));
|
||||||
|
|
||||||
|
final List<PathMetadata> oldMetas = numDelete < 0 ? null :
|
||||||
|
new ArrayList<>(numDelete);
|
||||||
|
for (int i = 0; i < numDelete; i++) {
|
||||||
|
oldMetas.add(new PathMetadata(
|
||||||
|
basicFileStatus(new Path(oldDir, "child" + i), i, true)));
|
||||||
|
}
|
||||||
|
final List<PathMetadata> newMetas = numPut < 0 ? null :
|
||||||
|
new ArrayList<>(numPut);
|
||||||
|
for (int i = 0; i < numPut; i++) {
|
||||||
|
newMetas.add(new PathMetadata(
|
||||||
|
basicFileStatus(new Path(newDir, "child" + i), i, false)));
|
||||||
|
}
|
||||||
|
|
||||||
|
Collection<Path> pathsToDelete = null;
|
||||||
|
if (oldMetas != null) {
|
||||||
|
// put all metadata of old paths and verify
|
||||||
|
ms.put(new DirListingMetadata(oldDir, oldMetas, false));
|
||||||
|
assertEquals(0, ms.listChildren(newDir).withoutTombstones().numEntries());
|
||||||
|
assertTrue(CollectionUtils
|
||||||
|
.isEqualCollection(oldMetas, ms.listChildren(oldDir).getListing()));
|
||||||
|
|
||||||
|
pathsToDelete = new ArrayList<>(oldMetas.size());
|
||||||
|
for (PathMetadata meta : oldMetas) {
|
||||||
|
pathsToDelete.add(meta.getFileStatus().getPath());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// move the old paths to new paths and verify
|
||||||
|
ms.move(pathsToDelete, newMetas);
|
||||||
|
assertEquals(0, ms.listChildren(oldDir).withoutTombstones().numEntries());
|
||||||
|
if (newMetas != null) {
|
||||||
|
assertTrue(CollectionUtils
|
||||||
|
.isEqualCollection(newMetas, ms.listChildren(newDir).getListing()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testInitExistingTable() throws IOException {
|
||||||
|
final DynamoDBMetadataStore ddbms = getDynamoMetadataStore();
|
||||||
|
final String tableName = ddbms.getTable().getTableName();
|
||||||
|
verifyTableInitialized(tableName, ddbms.getDynamoDB());
|
||||||
|
// create existing table
|
||||||
|
ddbms.initTable();
|
||||||
|
verifyTableInitialized(tableName, ddbms.getDynamoDB());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test the low level version check code.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void testItemVersionCompatibility() throws Throwable {
|
||||||
|
verifyVersionCompatibility("table",
|
||||||
|
createVersionMarker(VERSION_MARKER, VERSION, 0));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test that a version marker entry without the version number field
|
||||||
|
* is rejected as incompatible with a meaningful error message.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void testItemLacksVersion() throws Throwable {
|
||||||
|
intercept(IOException.class, E_NOT_VERSION_MARKER,
|
||||||
|
() -> verifyVersionCompatibility("table",
|
||||||
|
new Item().withPrimaryKey(
|
||||||
|
createVersionMarkerPrimaryKey(VERSION_MARKER))));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete the version marker and verify that table init fails.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void testTableVersionRequired() throws Exception {
|
||||||
|
String tableName = "testTableVersionRequired";
|
||||||
|
Configuration conf = getFileSystem().getConf();
|
||||||
|
int maxRetries = conf.getInt(S3GUARD_DDB_MAX_RETRIES,
|
||||||
|
S3GUARD_DDB_MAX_RETRIES_DEFAULT);
|
||||||
|
conf.setInt(S3GUARD_DDB_MAX_RETRIES, 3);
|
||||||
|
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName);
|
||||||
|
|
||||||
|
try(DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) {
|
||||||
|
ddbms.initialize(conf);
|
||||||
|
Table table = verifyTableInitialized(tableName, ddbms.getDynamoDB());
|
||||||
|
table.deleteItem(VERSION_MARKER_PRIMARY_KEY);
|
||||||
|
|
||||||
|
// create existing table
|
||||||
|
intercept(IOException.class, E_NO_VERSION_MARKER,
|
||||||
|
() -> ddbms.initTable());
|
||||||
|
|
||||||
|
conf.setInt(S3GUARD_DDB_MAX_RETRIES, maxRetries);
|
||||||
|
ddbms.destroy();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Set the version value to a different number and verify that
|
||||||
|
* table init fails.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void testTableVersionMismatch() throws Exception {
|
||||||
|
String tableName = "testTableVersionMismatch";
|
||||||
|
Configuration conf = getFileSystem().getConf();
|
||||||
|
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName);
|
||||||
|
|
||||||
|
try(DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) {
|
||||||
|
ddbms.initialize(conf);
|
||||||
|
Table table = verifyTableInitialized(tableName, ddbms.getDynamoDB());
|
||||||
|
table.deleteItem(VERSION_MARKER_PRIMARY_KEY);
|
||||||
|
Item v200 = createVersionMarker(VERSION_MARKER, 200, 0);
|
||||||
|
table.putItem(v200);
|
||||||
|
|
||||||
|
// create existing table
|
||||||
|
intercept(IOException.class, E_INCOMPATIBLE_VERSION,
|
||||||
|
() -> ddbms.initTable());
|
||||||
|
ddbms.destroy();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test that initTable fails with IOException when table does not exist and
|
||||||
|
* table auto-creation is disabled.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void testFailNonexistentTable() throws IOException {
|
||||||
|
final String tableName = "testFailNonexistentTable";
|
||||||
|
final S3AFileSystem s3afs = getFileSystem();
|
||||||
|
final Configuration conf = s3afs.getConf();
|
||||||
|
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName);
|
||||||
|
conf.unset(S3GUARD_DDB_TABLE_CREATE_KEY);
|
||||||
|
try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) {
|
||||||
|
ddbms.initialize(s3afs);
|
||||||
|
fail("Should have failed as table does not exist and table auto-creation"
|
||||||
|
+ " is disabled");
|
||||||
|
} catch (IOException ignored) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test cases about root directory as it is not in the DynamoDB table.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void testRootDirectory() throws IOException {
|
||||||
|
final DynamoDBMetadataStore ddbms = getDynamoMetadataStore();
|
||||||
|
Path rootPath = new Path(new Path(fsUri), "/");
|
||||||
|
verifyRootDirectory(ddbms.get(rootPath), true);
|
||||||
|
|
||||||
|
ddbms.put(new PathMetadata(new S3AFileStatus(true,
|
||||||
|
new Path(rootPath, "foo"),
|
||||||
|
UserGroupInformation.getCurrentUser().getShortUserName())));
|
||||||
|
verifyRootDirectory(ddbms.get(rootPath), false);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void verifyRootDirectory(PathMetadata rootMeta, boolean isEmpty) {
|
||||||
|
assertNotNull(rootMeta);
|
||||||
|
final FileStatus status = rootMeta.getFileStatus();
|
||||||
|
assertNotNull(status);
|
||||||
|
assertTrue(status.isDirectory());
|
||||||
|
// UNKNOWN is always a valid option, but true / false should not contradict
|
||||||
|
if (isEmpty) {
|
||||||
|
assertNotSame("Should not be marked non-empty",
|
||||||
|
Tristate.FALSE,
|
||||||
|
rootMeta.isEmptyDirectory());
|
||||||
|
} else {
|
||||||
|
assertNotSame("Should not be marked empty",
|
||||||
|
Tristate.TRUE,
|
||||||
|
rootMeta.isEmptyDirectory());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test that when moving nested paths, all its ancestors up to destination
|
||||||
|
* root will also be created.
|
||||||
|
* Here is the directory tree before move:
|
||||||
|
* <pre>
|
||||||
|
* testMovePopulateAncestors
|
||||||
|
* ├── a
|
||||||
|
* │ └── b
|
||||||
|
* │ └── src
|
||||||
|
* │ ├── dir1
|
||||||
|
* │ │ └── dir2
|
||||||
|
* │ └── file1.txt
|
||||||
|
* └── c
|
||||||
|
* └── d
|
||||||
|
* └── dest
|
||||||
|
*</pre>
|
||||||
|
* As part of rename(a/b/src, d/c/dest), S3A will enumerate the subtree at
|
||||||
|
* a/b/src. This test verifies that after the move, the new subtree at
|
||||||
|
* 'dest' is reachable from the root (i.e. c/ and c/d exist in the table.
|
||||||
|
* DynamoDBMetadataStore depends on this property to do recursive delete
|
||||||
|
* without a full table scan.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void testMovePopulatesAncestors() throws IOException {
|
||||||
|
final DynamoDBMetadataStore ddbms = getDynamoMetadataStore();
|
||||||
|
final String testRoot = "/testMovePopulatesAncestors";
|
||||||
|
final String srcRoot = testRoot + "/a/b/src";
|
||||||
|
final String destRoot = testRoot + "/c/d/e/dest";
|
||||||
|
|
||||||
|
final Path nestedPath1 = strToPath(srcRoot + "/file1.txt");
|
||||||
|
ddbms.put(new PathMetadata(basicFileStatus(nestedPath1, 1024, false)));
|
||||||
|
final Path nestedPath2 = strToPath(srcRoot + "/dir1/dir2");
|
||||||
|
ddbms.put(new PathMetadata(basicFileStatus(nestedPath2, 0, true)));
|
||||||
|
|
||||||
|
// We don't put the destRoot path here, since put() would create ancestor
|
||||||
|
// entries, and we want to ensure that move() does it, instead.
|
||||||
|
|
||||||
|
// Build enumeration of src / dest paths and do the move()
|
||||||
|
final Collection<Path> fullSourcePaths = Lists.newArrayList(
|
||||||
|
strToPath(srcRoot),
|
||||||
|
strToPath(srcRoot + "/dir1"),
|
||||||
|
strToPath(srcRoot + "/dir1/dir2"),
|
||||||
|
strToPath(srcRoot + "/file1.txt")
|
||||||
|
);
|
||||||
|
final Collection<PathMetadata> pathsToCreate = Lists.newArrayList(
|
||||||
|
new PathMetadata(basicFileStatus(strToPath(destRoot),
|
||||||
|
0, true)),
|
||||||
|
new PathMetadata(basicFileStatus(strToPath(destRoot + "/dir1"),
|
||||||
|
0, true)),
|
||||||
|
new PathMetadata(basicFileStatus(strToPath(destRoot + "/dir1/dir2"),
|
||||||
|
0, true)),
|
||||||
|
new PathMetadata(basicFileStatus(strToPath(destRoot + "/file1.txt"),
|
||||||
|
1024, false))
|
||||||
|
);
|
||||||
|
|
||||||
|
ddbms.move(fullSourcePaths, pathsToCreate);
|
||||||
|
|
||||||
|
// assert that all the ancestors should have been populated automatically
|
||||||
|
assertCached(testRoot + "/c");
|
||||||
|
assertCached(testRoot + "/c/d");
|
||||||
|
assertCached(testRoot + "/c/d/e");
|
||||||
|
assertCached(destRoot /* /c/d/e/dest */);
|
||||||
|
|
||||||
|
// Also check moved files while we're at it
|
||||||
|
assertCached(destRoot + "/dir1");
|
||||||
|
assertCached(destRoot + "/dir1/dir2");
|
||||||
|
assertCached(destRoot + "/file1.txt");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testProvisionTable() throws IOException {
|
||||||
|
final String tableName = "testProvisionTable";
|
||||||
|
Configuration conf = getFileSystem().getConf();
|
||||||
|
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName);
|
||||||
|
|
||||||
|
try(DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) {
|
||||||
|
ddbms.initialize(conf);
|
||||||
|
DynamoDB dynamoDB = ddbms.getDynamoDB();
|
||||||
|
final ProvisionedThroughputDescription oldProvision =
|
||||||
|
dynamoDB.getTable(tableName).describe().getProvisionedThroughput();
|
||||||
|
ddbms.provisionTable(oldProvision.getReadCapacityUnits() * 2,
|
||||||
|
oldProvision.getWriteCapacityUnits() * 2);
|
||||||
|
ddbms.initTable();
|
||||||
|
final ProvisionedThroughputDescription newProvision =
|
||||||
|
dynamoDB.getTable(tableName).describe().getProvisionedThroughput();
|
||||||
|
LOG.info("Old provision = {}, new provision = {}", oldProvision,
|
||||||
|
newProvision);
|
||||||
|
assertEquals(oldProvision.getReadCapacityUnits() * 2,
|
||||||
|
newProvision.getReadCapacityUnits().longValue());
|
||||||
|
assertEquals(oldProvision.getWriteCapacityUnits() * 2,
|
||||||
|
newProvision.getWriteCapacityUnits().longValue());
|
||||||
|
ddbms.destroy();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testDeleteTable() throws Exception {
|
||||||
|
final String tableName = "testDeleteTable";
|
||||||
|
Path testPath = new Path(new Path(fsUri), "/" + tableName);
|
||||||
|
final S3AFileSystem s3afs = getFileSystem();
|
||||||
|
final Configuration conf = s3afs.getConf();
|
||||||
|
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, tableName);
|
||||||
|
try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) {
|
||||||
|
ddbms.initialize(s3afs);
|
||||||
|
// we can list the empty table
|
||||||
|
ddbms.listChildren(testPath);
|
||||||
|
DynamoDB dynamoDB = ddbms.getDynamoDB();
|
||||||
|
ddbms.destroy();
|
||||||
|
verifyTableNotExist(tableName, dynamoDB);
|
||||||
|
|
||||||
|
// delete table once more; be ResourceNotFoundException swallowed silently
|
||||||
|
ddbms.destroy();
|
||||||
|
verifyTableNotExist(tableName, dynamoDB);
|
||||||
|
try {
|
||||||
|
// we can no longer list the destroyed table
|
||||||
|
ddbms.listChildren(testPath);
|
||||||
|
fail("Should have failed after the table is destroyed!");
|
||||||
|
} catch (IOException ignored) {
|
||||||
|
}
|
||||||
|
ddbms.destroy();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This validates the table is created and ACTIVE in DynamoDB.
|
||||||
|
*
|
||||||
|
* This should not rely on the {@link DynamoDBMetadataStore} implementation.
|
||||||
|
* Return the table
|
||||||
|
*/
|
||||||
|
private Table verifyTableInitialized(String tableName, DynamoDB dynamoDB) {
|
||||||
|
final Table table = dynamoDB.getTable(tableName);
|
||||||
|
final TableDescription td = table.describe();
|
||||||
|
assertEquals(tableName, td.getTableName());
|
||||||
|
assertEquals("ACTIVE", td.getTableStatus());
|
||||||
|
return table;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This validates the table is not found in DynamoDB.
|
||||||
|
*
|
||||||
|
* This should not rely on the {@link DynamoDBMetadataStore} implementation.
|
||||||
|
*/
|
||||||
|
private void verifyTableNotExist(String tableName, DynamoDB dynamoDB) throws
|
||||||
|
Exception{
|
||||||
|
intercept(ResourceNotFoundException.class,
|
||||||
|
() -> dynamoDB.getTable(tableName).describe());
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user