HADOOP-16953. tuning s3guard disabled warnings (#1962)

Contributed by Steve Loughran.

The S3Guard absence warning of HADOOP-16484 has been changed
so that by default the S3A connector only logs at debug
when the connection to the S3 Store does not have S3Guard
enabled.

The option to control this log level is now
fs.s3a.s3guard.disabled.warn.level
and can be one of: silent, inform, warn, fail.

On a failure, an ExitException is raised with exit code 49.

For details on this safety feature, consult the s3guard documentation.

Change-Id: If868671c9260977c2b03b3e475b9c9531c98ce79

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java

@@ -746,9 +746,9 @@ private Constants() {
    * The warn level if S3Guard is disabled.
    */
   public static final String S3GUARD_DISABLED_WARN_LEVEL
-      = "org.apache.hadoop.fs.s3a.s3guard.disabled.warn.level";
+      = "fs.s3a.s3guard.disabled.warn.level";
   public static final String DEFAULT_S3GUARD_DISABLED_WARN_LEVEL =
-      "INFORM";
+      "SILENT";
 
   /**
    * Inconsistency (visibility delay) injection settings.

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java

@@ -53,12 +53,14 @@
 import org.apache.hadoop.fs.s3a.Retries.RetryTranslated;
 import org.apache.hadoop.fs.s3a.S3AFileStatus;
 import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.util.ExitUtil;
 import org.apache.hadoop.util.ReflectionUtils;
 
 import static org.apache.hadoop.fs.s3a.Constants.*;
 import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_AUTHORITATIVE_PATH;
 import static org.apache.hadoop.fs.s3a.S3AUtils.createUploadFileStatus;
 import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.authoritativeEmptyDirectoryMarker;
+import static org.apache.hadoop.service.launcher.LauncherExitCodes.EXIT_BAD_CONFIGURATION;
 
 /**
  * Logic for integrating MetadataStore with S3A.
@@ -1041,12 +1043,22 @@ public static boolean allowAuthoritative(Path p, S3AFileSystem fs,
     return false;
   }
 
+  /**
+   * Format string to use when warning that S3Guard is disabled.
+   */
+  @VisibleForTesting
   public static final String DISABLED_LOG_MSG =
-      "S3Guard is disabled on this bucket: {}";
+      "S3Guard is disabled on this bucket: %s";
 
+  /**
+   * Error string use in exception raised on an unknown log level.
+   */
   public static final String UNKNOWN_WARN_LEVEL =
-      "Unknown S3Guard disabled warn level: ";
+      "Unknown " + S3GUARD_DISABLED_WARN_LEVEL + " value: ";
 
+  /**
+   * Warning levels to use when reporting S3Guard as disabled.
+   */
   public enum DisabledWarnLevel {
     SILENT,
     INFORM,
@@ -1054,9 +1066,18 @@ public enum DisabledWarnLevel {
     FAIL
   }
 
+  /**
+   * Log that S3Guard is disabled -optionally raise an exception.
+   * @param logger Log to log to
+   * @param warnLevelStr string value of warn action.
+   * @param bucket bucket to use in log/error messages
+   * @throws ExitUtil.ExitException if s3guard was disabled
+   *                                and the log level is "fail"
+   * @throws IllegalArgumentException unknown warning level.
+   */
   public static void logS3GuardDisabled(Logger logger, String warnLevelStr,
       String bucket)
-      throws UnsupportedOperationException, IllegalArgumentException {
+      throws ExitUtil.ExitException, IllegalArgumentException {
     final DisabledWarnLevel warnLevel;
     try {
       warnLevel = DisabledWarnLevel.valueOf(warnLevelStr.toUpperCase(Locale.US));
@@ -1064,19 +1085,20 @@ public static void logS3GuardDisabled(Logger logger, String warnLevelStr,
       throw new IllegalArgumentException(UNKNOWN_WARN_LEVEL + warnLevelStr, e);
     }
 
+    String text = String.format(DISABLED_LOG_MSG, bucket);
     switch (warnLevel) {
     case SILENT:
-      logger.debug(DISABLED_LOG_MSG, bucket);
+      logger.debug(text);
       break;
     case INFORM:
-      logger.info(DISABLED_LOG_MSG, bucket);
+      logger.info(text);
       break;
     case WARN:
-      logger.warn(DISABLED_LOG_MSG, bucket);
+      logger.warn(text);
       break;
     case FAIL:
-      logger.error(DISABLED_LOG_MSG, bucket);
+      logger.error(text);
-      throw new UnsupportedOperationException(DISABLED_LOG_MSG + bucket);
+      throw new ExitUtil.ExitException(EXIT_BAD_CONFIGURATION, text);
     default:
       throw new IllegalArgumentException(UNKNOWN_WARN_LEVEL + warnLevelStr);
     }

hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md

@@ -1236,6 +1236,35 @@ Deleting the metadata store table will simply result in a period of eventual
 consistency for any file modifications that were made right before the table
 was deleted.
 
+### Enabling a log message whenever S3Guard is *disabled*
+
+When dealing with support calls related to the S3A connector, "is S3Guard on?"
+is the usual opening question. This can be determined by looking at the application logs for
+messages about S3Guard starting -the absence of S3Guard can only be inferred by the absence
+of such messages.
+
+There is a another strategy: have the S3A Connector log whenever *S3Guard is not enabled*
+
+This can be done in the configuration option `fs.s3a.s3guard.disabled.warn.level`
+
+```xml
+<property>
+ <name>fs.s3a.s3guard.disabled.warn.level</name>
+ <value>silent</value>
+ <description>
+   Level to print a message when S3Guard is disabled.
+   Values: 
+   "warn": log at WARN level
+   "inform": log at INFO level
+   "silent": log at DEBUG level
+   "fail": raise an exception
+ </description>
+</property>
+```
+
+The `fail` option is clearly more than logging; it exists as an extreme debugging
+tool. Use with care.
+
 ### Failure Semantics
 
 Operations which modify metadata will make changes to S3 first. If, and only

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3Guard.java

@@ -41,7 +41,9 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.s3a.S3AFileStatus;
 import org.apache.hadoop.fs.s3a.Tristate;
+import org.apache.hadoop.service.launcher.LauncherExitCodes;
 import org.apache.hadoop.test.LambdaTestUtils;
+import org.apache.hadoop.util.ExitUtil;
 
 import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_METADATASTORE_METADATA_TTL;
 import static org.apache.hadoop.fs.s3a.Constants.METADATASTORE_METADATA_TTL;
@@ -429,11 +431,17 @@ public void testLogS3GuardDisabled() throws Exception {
         S3Guard.DisabledWarnLevel.WARN.toString()
             .toLowerCase(Locale.US), "bucket");
 
-    LambdaTestUtils.intercept(UnsupportedOperationException.class,
+    ExitUtil.ExitException ex = LambdaTestUtils.intercept(
-        S3Guard.DISABLED_LOG_MSG, () -> S3Guard.logS3GuardDisabled(
+        ExitUtil.ExitException.class,
+        String.format(S3Guard.DISABLED_LOG_MSG, "bucket"),
+        () -> S3Guard.logS3GuardDisabled(
             localLogger, S3Guard.DisabledWarnLevel.FAIL.toString(), "bucket"));
+    if (ex.getExitCode() != LauncherExitCodes.EXIT_BAD_CONFIGURATION) {
+      throw ex;
+    }
     LambdaTestUtils.intercept(IllegalArgumentException.class,
-        S3Guard.UNKNOWN_WARN_LEVEL, () -> S3Guard.logS3GuardDisabled(
+        S3Guard.UNKNOWN_WARN_LEVEL,
+        () -> S3Guard.logS3GuardDisabled(
             localLogger, "FOO_BAR_LEVEL", "bucket"));
   }