big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-15645. ITestS3GuardToolLocal.testDiffCommand fails if bucket has per-bucket binding to DDB. Contributed by Steve Loughran.

Browse Source
This commit is contained in:
Ewan Higgs 2018-08-13 12:57:45 +02:00
parent 475bff6e8e
commit a13929ddcb
3 changed files with 25 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
View File

@ -56,6 +56,7 @@
import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_NAME_KEY;
import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_METASTORE_NULL;
import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL;
import static org.apache.hadoop.fs.s3a.S3AUtils.clearBucketOption;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.E_BAD_STATE;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.SUCCESS;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
@ -142,12 +143,14 @@ protected MetadataStore getMetadataStore() {
public void setup() throws Exception {
super.setup();
S3ATestUtils.assumeS3GuardState(true, getConfiguration());
ms = getFileSystem().getMetadataStore();
S3AFileSystem fs = getFileSystem();
ms = fs.getMetadataStore();
// Also create a "raw" fs without any MetadataStore configured
Configuration conf = new Configuration(getConfiguration());
URI fsUri = getFileSystem().getUri();
clearBucketOption(conf, fs.getBucket(), S3_METADATA_STORE_IMPL);
conf.set(S3_METADATA_STORE_IMPL, S3GUARD_METASTORE_NULL);
URI fsUri = fs.getUri();
S3AUtils.setBucketOption(conf,fsUri.getHost(),
METADATASTORE_AUTHORITATIVE,
S3GUARD_METASTORE_NULL);
@ -394,13 +397,17 @@ protected void exec(S3GuardTool cmd, ByteArrayOutputStream buf, String...args)
}
@Test
public void testDiffCommand() throws Exception {
public void
testDiffCommand() throws Exception {
S3AFileSystem fs = getFileSystem();
ms = getMetadataStore();
Set<Path> filesOnS3 = new HashSet<>(); // files on S3.
Set<Path> filesOnMS = new HashSet<>(); // files on metadata store.
Path testPath = path("test-diff");
// clean up through the store and behind it.
fs.delete(testPath, true);
rawFs.delete(testPath, true);
mkdirs(testPath, true, true);
Path msOnlyPath = new Path(testPath, "ms_only");

7
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
View File

@ -50,9 +50,10 @@ public class ITestS3GuardToolDynamoDB extends AbstractS3GuardToolTestBase {
@Override
public void setup() throws Exception {
super.setup();
Assume.assumeTrue("Test only applies when DynamoDB is used for S3Guard",
getConfiguration().get(Constants.S3_METADATA_STORE_IMPL).equals(
Constants.S3GUARD_METASTORE_DYNAMO));
MetadataStore ms = getMetadataStore();
Assume.assumeTrue("Test only applies when DynamoDB is used for S3Guard;"
+ "Store is " + (ms == null ? "none" : ms.toString()),
ms instanceof DynamoDBMetadataStore);
}
// Check the existence of a given DynamoDB table.

11
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
View File

@ -31,6 +31,8 @@
import org.apache.hadoop.test.LambdaTestUtils;
import org.apache.hadoop.util.StringUtils;
import org.junit.Assume;
import org.junit.Test;
import org.apache.hadoop.fs.FSDataOutputStream;
@ -52,6 +54,15 @@ public class ITestS3GuardToolLocal extends AbstractS3GuardToolTestBase {
private static final String[] ABORT_FORCE_OPTIONS = new String[] {"-abort",
"-force", "-verbose"};
@Override
public void setup() throws Exception {
super.setup();
MetadataStore ms = getMetadataStore();
Assume.assumeTrue("Test only applies when a local store is used for S3Guard;"
+ "Store is " + (ms == null ? "none" : ms.toString()),
ms instanceof LocalMetadataStore);
}
@Test
public void testImportCommand() throws Exception {
S3AFileSystem fs = getFileSystem();