HADOOP-15843. s3guard bucket-info command to not print a stack trace on bucket-not-found.

Contributed by Adam Antal.

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java

@@ -66,6 +66,7 @@
  * CLI to manage S3Guard Metadata Store.
  */
 public abstract class S3GuardTool extends Configured implements Tool {
+
   private static final Logger LOG = LoggerFactory.getLogger(S3GuardTool.class);
 
   private static final String NAME = "s3guard";
@@ -91,6 +92,9 @@ public abstract class S3GuardTool extends Configured implements Tool {
   private static final String DATA_IN_S3_IS_PRESERVED
       = "(all data in S3 is preserved)";
 
+  public static final String E_NO_METASTORE_OR_FILESYSTEM
+      = "No metastore or filesystem specified";
+
   abstract public String getUsage();
 
   // Exit codes
@@ -267,12 +271,8 @@ MetadataStore initMetadataStore(boolean forceCreate) throws IOException {
     if (getStore() != null) {
       return getStore();
     }
-    Configuration conf;
+    final boolean hasFileSystem = filesystem != null;
-    if (filesystem == null) {
+    final Configuration conf = hasFileSystem ? filesystem.getConf() : getConf();
-      conf = getConf();
-    } else {
-      conf = filesystem.getConf();
-    }
     String metaURI = getCommandFormat().getOptValue(META_FLAG);
     if (metaURI != null && !metaURI.isEmpty()) {
       URI uri = URI.create(metaURI);
@@ -294,6 +294,13 @@ MetadataStore initMetadataStore(boolean forceCreate) throws IOException {
             String.format("Metadata store %s is not supported", uri));
       }
     } else {
+      if (!hasFileSystem) {
+        // command didn't declare a metadata store URI or a bucket.
+        // to avoid problems related to picking up a shared table for actions
+        // line init and destroy (HADOOP-15843), this is rejected
+        printHelp(this);
+        throw usageError(E_NO_METASTORE_OR_FILESYSTEM);
+      }
       // CLI does not specify metadata store URI, it uses default metadata store
       // DynamoDB instead.
       setStore(new DynamoDBMetadataStore());
@@ -302,10 +309,10 @@ MetadataStore initMetadataStore(boolean forceCreate) throws IOException {
       }
     }
 
-    if (filesystem == null) {
+    if (hasFileSystem) {
-      getStore().initialize(conf);
-    } else {
       getStore().initialize(filesystem);
+    } else {
+      getStore().initialize(conf);
     }
     LOG.info("Metadata store {} is initialized.", getStore());
     return getStore();
@@ -1431,13 +1438,13 @@ protected static URI toUri(String s3Path) {
     return uri;
   }
 
-  private static void printHelp() {
+  private static void printHelp(S3GuardTool tool) {
-    if (command == null) {
+    if (tool == null) {
       errorln("Usage: hadoop " + USAGE);
       errorln("\tperform S3Guard metadata store " +
           "administrative commands.");
     } else {
-      errorln("Usage: hadoop " + command.getUsage());
+      errorln("Usage: hadoop " + tool.getUsage());
     }
     errorln();
     errorln(COMMON_USAGE);
@@ -1477,7 +1484,6 @@ protected static void printStoreDiagnostics(PrintStream out,
     }
   }
 
-
   /**
    * Handle store not found by converting to an exit exception
    * with specific error code.
@@ -1525,6 +1531,18 @@ protected static ExitUtil.ExitException userAborted(
     return new ExitUtil.ExitException(ERROR, String.format(format, args));
   }
 
+
+  /**
+   * Build the exception to raise on a usage error
+   * @param format string format
+   * @param args optional arguments for the string
+   * @return a new exception to throw
+   */
+  protected static ExitUtil.ExitException usageError(
+      String format, Object...args) {
+    return new ExitUtil.ExitException(E_USAGE, String.format(format, args));
+  }
+
   /**
    * Execute the command with the given arguments.
    *
@@ -1540,8 +1558,8 @@ public static int run(Configuration conf, String...args) throws
     String[] otherArgs = new GenericOptionsParser(conf, args)
         .getRemainingArgs();
     if (otherArgs.length == 0) {
-      printHelp();
+      printHelp(null);
-      throw new ExitUtil.ExitException(E_USAGE, "No arguments provided");
+      throw usageError("No arguments provided");
     }
     final String subCommand = otherArgs[0];
     LOG.debug("Executing command {}", subCommand);
@@ -1571,7 +1589,7 @@ public static int run(Configuration conf, String...args) throws
       command = new Uploads(conf);
       break;
     default:
-      printHelp();
+      printHelp(null);
       throw new ExitUtil.ExitException(E_USAGE,
           "Unknown command " + subCommand);
     }
@@ -1588,11 +1606,17 @@ public static void main(String[] args) {
       exit(ret, "");
     } catch (CommandFormat.UnknownOptionException e) {
       errorln(e.getMessage());
-      printHelp();
+      printHelp(command);
       exit(E_USAGE, e.getMessage());
     } catch (ExitUtil.ExitException e) {
       // explicitly raised exit code
       exit(e.getExitCode(), e.toString());
+    } catch (FileNotFoundException e) {
+      // bucket doesn't exist or similar.
+      // skip the stack trace and choose the return code of 44, "404"
+      errorln(e.toString());
+      LOG.debug("Not found:", e);
+      exit(EXIT_NOT_FOUND, e.toString());
     } catch (Throwable e) {
       e.printStackTrace(System.err);
       exit(ERROR, e.toString());

hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md

@@ -1006,6 +1006,20 @@ There's are limit on how often you can change the capacity of an DynamoDB table;
 if you call set-capacity too often, it fails. Wait until the after the time indicated
 and try again.
 
+### Error `Invalid region specified`
+
+```
+java.io.IOException: Invalid region specified "iceland-2":
+  Region can be configured with fs.s3a.s3guard.ddb.region:
+  us-gov-west-1, us-east-1, us-east-2, us-west-1, us-west-2,
+  eu-west-1, eu-west-2, eu-west-3, eu-central-1, ap-south-1,
+  ap-southeast-1, ap-southeast-2, ap-northeast-1, ap-northeast-2,
+  sa-east-1, cn-north-1, cn-northwest-1, ca-central-1
+  at org.apache.hadoop.fs.s3a.s3guard.DynamoDBClientFactory$DefaultDynamoDBClientFactory.getRegion
+  at org.apache.hadoop.fs.s3a.s3guard.DynamoDBClientFactory$DefaultDynamoDBClientFactory.createDynamoDBClient
+```
+
+The region specified in `fs.s3a.s3guard.ddb.region` is invalid.
 
 ## Other Topics
 

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java

@@ -54,12 +54,15 @@
 import org.apache.hadoop.util.StringUtils;
 
 import static org.apache.hadoop.fs.s3a.Constants.METADATASTORE_AUTHORITATIVE;
+import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_REGION_KEY;
 import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_CREATE_KEY;
 import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_NAME_KEY;
 import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_METASTORE_NULL;
 import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL;
 import static org.apache.hadoop.fs.s3a.S3AUtils.clearBucketOption;
 import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.E_BAD_STATE;
+import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.E_NO_METASTORE_OR_FILESYSTEM;
+import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.E_USAGE;
 import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.SUCCESS;
 import static org.apache.hadoop.test.LambdaTestUtils.intercept;
 
@@ -337,28 +340,68 @@ public void testBucketInfoUnguarded() throws Exception {
   @Test
   public void testSetCapacityFailFastIfNotGuarded() throws Exception{
     Configuration conf = getConfiguration();
-    conf.set(S3GUARD_DDB_TABLE_NAME_KEY, UUID.randomUUID().toString());
+    bindToNonexistentTable(conf);
-    conf.set(S3GUARD_DDB_TABLE_CREATE_KEY, Boolean.FALSE.toString());
+    String bucket = rawFs.getBucket();
+    clearBucketOption(conf, bucket, S3_METADATA_STORE_IMPL);
+    clearBucketOption(conf, bucket, S3GUARD_DDB_TABLE_NAME_KEY);
+    clearBucketOption(conf, bucket, S3GUARD_DDB_TABLE_CREATE_KEY);
     conf.set(S3_METADATA_STORE_IMPL, S3GUARD_METASTORE_NULL);
 
     S3GuardTool.SetCapacity cmdR = new S3GuardTool.SetCapacity(conf);
-    String[] argsR = new String[]{cmdR.getName(),
+    String[] argsR = new String[]{
-        "s3a://" + getFileSystem().getBucket()};
+        cmdR.getName(),
+        "s3a://" + getFileSystem().getBucket()
+    };
 
     intercept(IllegalStateException.class, "unguarded",
-        () -> run(argsR));
+        () -> cmdR.run(argsR));
+  }
+
+  /**
+   * Binds the configuration to a nonexistent table.
+   * @param conf
+   */
+  protected void bindToNonexistentTable(final Configuration conf) {
+    conf.set(S3GUARD_DDB_TABLE_NAME_KEY, UUID.randomUUID().toString());
+    conf.setBoolean(S3GUARD_DDB_TABLE_CREATE_KEY, false);
   }
 
   @Test
   public void testDestroyNoBucket() throws Throwable {
+    describe("Destroy a bucket which doesn't exist");
+
+    Configuration conf = getConfiguration();
+    // set a table as a safety check in case the test goes wrong
+    // and deletes it.
+    bindToNonexistentTable(conf);
+
+    S3GuardTool.Destroy cmdR = new S3GuardTool.Destroy(conf);
+    String[] argsR = new String[]{
+        S3GuardTool.Destroy.NAME,
+        S3A_THIS_BUCKET_DOES_NOT_EXIST
+    };
     intercept(FileNotFoundException.class,
-        new Callable<Integer>() {
+        () -> cmdR.run(argsR));
-          @Override
+  }
-          public Integer call() throws Exception {
+
-            return run(S3GuardTool.Destroy.NAME,
+  @Test
-                S3A_THIS_BUCKET_DOES_NOT_EXIST);
+  public void testDestroyNoArgs() throws Throwable {
-          }
+    describe("Destroy a bucket which doesn't exist");
-        });
+
+    Configuration conf = getConfiguration();
+    // set a table as a safety check in case the test goes wrong
+    // and deletes it.
+    conf.set(S3GUARD_DDB_TABLE_NAME_KEY, UUID.randomUUID().toString());
+    conf.set(S3GUARD_DDB_REGION_KEY, "us-gov-west-1");
+    conf.setBoolean(S3GUARD_DDB_TABLE_CREATE_KEY, false);
+
+    S3GuardTool.Destroy cmdR = new S3GuardTool.Destroy(conf);
+
+    assertExitCode(E_USAGE,
+        intercept(ExitUtil.ExitException.class,
+            E_NO_METASTORE_OR_FILESYSTEM,
+            () -> cmdR.run(new String[]{})));
+
   }
 
   @Test
@@ -373,11 +416,24 @@ public void testProbeForMagic() throws Throwable {
       exec(cmd, S3GuardTool.BucketInfo.MAGIC_FLAG, name);
     } else {
       // if the FS isn't magic, expect the probe to fail
-      ExitUtil.ExitException e = intercept(ExitUtil.ExitException.class,
+      assertExitCode(E_BAD_STATE,
-          () -> exec(cmd, S3GuardTool.BucketInfo.MAGIC_FLAG, name));
+          intercept(ExitUtil.ExitException.class,
-      if (e.getExitCode() != E_BAD_STATE) {
+              () -> exec(cmd, S3GuardTool.BucketInfo.MAGIC_FLAG, name)));
-        throw e;
+    }
-      }
+  }
+
+  /**
+   * Assert that an exit exception had a specific error code.
+   * @param expectedErrorCode expected code.
+   * @param e exit exception
+   * @throws AssertionError with the exit exception nested inside
+   */
+  protected void assertExitCode(final int expectedErrorCode,
+      final ExitUtil.ExitException e) {
+    if (e.getExitCode() != expectedErrorCode) {
+      throw new AssertionError("Expected error code " + expectedErrorCode
+          + " in " + e,
+          e);
     }
   }