From a13929ddcb3b90044350ae1c23a1150e8b4b975b Mon Sep 17 00:00:00 2001
From: Ewan Higgs <ewan.higgs@wdc.com>
Date: Mon, 13 Aug 2018 12:57:45 +0200
Subject: [PATCH] HADOOP-15645. ITestS3GuardToolLocal.testDiffCommand fails if
 bucket has per-bucket binding to DDB. Contributed by Steve Loughran.

---
 .../fs/s3a/s3guard/AbstractS3GuardToolTestBase.java | 13 ++++++++++---
 .../fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java    |  7 ++++---
 .../fs/s3a/s3guard/ITestS3GuardToolLocal.java       | 11 +++++++++++
 3 files changed, 25 insertions(+), 6 deletions(-)

diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
index 9185fc5b7c..96aac15fc0 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
@@ -56,6 +56,7 @@
 import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_NAME_KEY;
 import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_METASTORE_NULL;
 import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL;
+import static org.apache.hadoop.fs.s3a.S3AUtils.clearBucketOption;
 import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.E_BAD_STATE;
 import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.SUCCESS;
 import static org.apache.hadoop.test.LambdaTestUtils.intercept;
@@ -142,12 +143,14 @@ protected MetadataStore getMetadataStore() {
   public void setup() throws Exception {
     super.setup();
     S3ATestUtils.assumeS3GuardState(true, getConfiguration());
-    ms = getFileSystem().getMetadataStore();
+    S3AFileSystem fs = getFileSystem();
+    ms = fs.getMetadataStore();
 
     // Also create a "raw" fs without any MetadataStore configured
     Configuration conf = new Configuration(getConfiguration());
-    URI fsUri = getFileSystem().getUri();
+    clearBucketOption(conf, fs.getBucket(), S3_METADATA_STORE_IMPL);
     conf.set(S3_METADATA_STORE_IMPL, S3GUARD_METASTORE_NULL);
+    URI fsUri = fs.getUri();
     S3AUtils.setBucketOption(conf,fsUri.getHost(),
         METADATASTORE_AUTHORITATIVE,
         S3GUARD_METASTORE_NULL);
@@ -394,13 +397,17 @@ protected void exec(S3GuardTool cmd, ByteArrayOutputStream buf, String...args)
   }
 
   @Test
-  public void testDiffCommand() throws Exception {
+  public void
+  testDiffCommand() throws Exception {
     S3AFileSystem fs = getFileSystem();
     ms = getMetadataStore();
     Set<Path> filesOnS3 = new HashSet<>(); // files on S3.
     Set<Path> filesOnMS = new HashSet<>(); // files on metadata store.
 
     Path testPath = path("test-diff");
+    // clean up through the store and behind it.
+    fs.delete(testPath, true);
+    rawFs.delete(testPath, true);
     mkdirs(testPath, true, true);
 
     Path msOnlyPath = new Path(testPath, "ms_only");
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
index 1a59bf1c93..c96cbd0ba3 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
@@ -50,9 +50,10 @@ public class ITestS3GuardToolDynamoDB extends AbstractS3GuardToolTestBase {
   @Override
   public void setup() throws Exception {
     super.setup();
-    Assume.assumeTrue("Test only applies when DynamoDB is used for S3Guard",
-        getConfiguration().get(Constants.S3_METADATA_STORE_IMPL).equals(
-            Constants.S3GUARD_METASTORE_DYNAMO));
+    MetadataStore ms = getMetadataStore();
+    Assume.assumeTrue("Test only applies when DynamoDB is used for S3Guard;"
+        + "Store is " + (ms == null ? "none" : ms.toString()),
+        ms instanceof DynamoDBMetadataStore);
   }
 
   // Check the existence of a given DynamoDB table.
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
index f5c4b0316b..1ee3cde80d 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
@@ -31,6 +31,8 @@
 
 import org.apache.hadoop.test.LambdaTestUtils;
 import org.apache.hadoop.util.StringUtils;
+
+import org.junit.Assume;
 import org.junit.Test;
 
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -52,6 +54,15 @@ public class ITestS3GuardToolLocal extends AbstractS3GuardToolTestBase {
   private static final String[] ABORT_FORCE_OPTIONS = new String[] {"-abort",
       "-force", "-verbose"};
 
+  @Override
+  public void setup() throws Exception {
+    super.setup();
+    MetadataStore ms = getMetadataStore();
+    Assume.assumeTrue("Test only applies when a local store is used for S3Guard;"
+            + "Store is " + (ms == null ? "none" : ms.toString()),
+        ms instanceof LocalMetadataStore);
+  }
+
   @Test
   public void testImportCommand() throws Exception {
     S3AFileSystem fs = getFileSystem();