HADOOP-18975 S3A: Add option fs.s3a.endpoint.fips to use AWS FIPS endpoints (#6277)

Adds a new option `fs.s3a.endpoint.fips` to switch the SDK client to use
FIPS endpoints, as an alternative to explicitly declaring them.


* The option is available as a path capability for probes.
* SDK v2 itself doesn't know that some regions don't have FIPS endpoints
* SDK only fails with endpoint + fips flag as a retried exception; wit this
  change the S3A client should fail fast.
  PR fails fast.
* Adds a new "connecting.md" doc; moves existing docs there and restructures.
* New Tests in ITestS3AEndpointRegion

bucket-info command support:

* added to list of path capabilities
* added -fips flag and test for explicit probe
* also now prints bucket region
* and removed some of the obsolete s3guard options
* updated docs

Contributed by Steve Loughran

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java

@@ -1335,6 +1335,15 @@ private Constants() {
    */
   public static final String AWS_S3_DEFAULT_REGION = "us-east-2";
 
+  /**
+   * Is the endpoint a FIPS endpoint?
+   * Can be queried as a path capability.
+   * Value {@value}.
+   */
+  public static final String FIPS_ENDPOINT = "fs.s3a.endpoint.fips";
+
+  public static final boolean ENDPOINT_FIPS_DEFAULT = false;
+
   /**
    * Require that all S3 access is made through Access Points.
    */

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java

@@ -22,6 +22,7 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.fs.s3a.impl.AWSClientConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -54,6 +55,7 @@
 import static org.apache.hadoop.fs.s3a.Constants.AWS_REGION;
 import static org.apache.hadoop.fs.s3a.Constants.AWS_S3_DEFAULT_REGION;
 import static org.apache.hadoop.fs.s3a.Constants.CENTRAL_ENDPOINT;
+import static org.apache.hadoop.fs.s3a.Constants.FIPS_ENDPOINT;
 import static org.apache.hadoop.fs.s3a.Constants.HTTP_SIGNER_CLASS_NAME;
 import static org.apache.hadoop.fs.s3a.Constants.HTTP_SIGNER_ENABLED;
 import static org.apache.hadoop.fs.s3a.Constants.HTTP_SIGNER_ENABLED_DEFAULT;
@@ -63,6 +65,7 @@
 import static org.apache.hadoop.fs.s3a.auth.SignerFactory.createHttpSigner;
 import static org.apache.hadoop.fs.s3a.impl.AWSHeaders.REQUESTER_PAYS_HEADER;
 import static org.apache.hadoop.fs.s3a.impl.InternalConstants.AUTH_SCHEME_AWS_SIGV_4;
+import static org.apache.hadoop.util.Preconditions.checkArgument;
 
 
 /**
@@ -102,6 +105,13 @@ public class DefaultS3ClientFactory extends Configured
   /** Exactly once log to inform about ignoring the AWS-SDK Warnings for CSE. */
   private static final LogExactlyOnce IGNORE_CSE_WARN = new LogExactlyOnce(LOG);
 
+  /**
+   * Error message when an endpoint is set with FIPS enabled: {@value}.
+   */
+  @VisibleForTesting
+  public static final String ERROR_ENDPOINT_WITH_FIPS =
+      "An endpoint cannot set when " + FIPS_ENDPOINT + " is true";
+
   @Override
   public S3Client createS3Client(
       final URI uri,
@@ -248,6 +258,7 @@ protected ClientOverrideConfiguration createClientOverrideConfiguration(
    * @param conf  conf configuration object
    * @param <BuilderT> S3 client builder type
    * @param <ClientT> S3 client type
+   * @throws IllegalArgumentException if endpoint is set when FIPS is enabled.
    */
   private <BuilderT extends S3BaseClientBuilder<BuilderT, ClientT>, ClientT> void configureEndpointAndRegion(
       BuilderT builder, S3ClientCreationParameters parameters, Configuration conf) {
@@ -263,7 +274,18 @@ private <BuilderT extends S3BaseClientBuilder<BuilderT, ClientT>, ClientT> void
       region = Region.of(configuredRegion);
     }
 
+    // FIPs? Log it, then reject any attempt to set an endpoint
+    final boolean fipsEnabled = parameters.isFipsEnabled();
+    if (fipsEnabled) {
+      LOG.debug("Enabling FIPS mode");
+    }
+    // always setting it guarantees the value is non-null,
+    // which tests expect.
+    builder.fipsEnabled(fipsEnabled);
+
     if (endpoint != null) {
+      checkArgument(!fipsEnabled,
+          "%s : %s", ERROR_ENDPOINT_WITH_FIPS, endpoint);
       builder.endpointOverride(endpoint);
       // No region was configured, try to determine it from the endpoint.
       if (region == null) {

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java

@@ -461,6 +461,11 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
    */
   private boolean isMultipartCopyEnabled;
 
+  /**
+   * Is FIPS enabled?
+   */
+  private boolean fipsEnabled;
+
   /**
    * A cache of files that should be deleted when the FileSystem is closed
    * or the JVM is exited.
@@ -614,6 +619,8 @@ public void initialize(URI name, Configuration originalConf)
           ? conf.getTrimmed(AWS_REGION)
           : accessPoint.getRegion();
 
+      fipsEnabled = conf.getBoolean(FIPS_ENDPOINT, ENDPOINT_FIPS_DEFAULT);
+
       // is this an S3Express store?
       s3ExpressStore = isS3ExpressStore(bucket, endpoint);
 
@@ -1046,6 +1053,7 @@ private void bindAWSClient(URI name, boolean dtEnabled) throws IOException {
         .withMultipartThreshold(multiPartThreshold)
         .withTransferManagerExecutor(unboundedThreadPool)
         .withRegion(configuredRegion)
+        .withFipsEnabled(fipsEnabled)
         .withExpressCreateSession(
             conf.getBoolean(S3EXPRESS_CREATE_SESSION, S3EXPRESS_CREATE_SESSION_DEFAULT));
 
@@ -5521,6 +5529,10 @@ public boolean hasPathCapability(final Path path, final String capability)
     case OPTIMIZED_COPY_FROM_LOCAL:
       return optimizedCopyFromLocal;
 
+    // probe for a fips endpoint
+    case FIPS_ENDPOINT:
+      return fipsEnabled;
+
     default:
       return super.hasPathCapability(p, cap);
     }

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java

@@ -176,6 +176,11 @@ final class S3ClientCreationParameters {
      */
     private boolean expressCreateSession = S3EXPRESS_CREATE_SESSION_DEFAULT;
 
+    /**
+     * Is FIPS enabled?
+     */
+    private boolean fipsEnabled;
+
     /**
      * List of execution interceptors to include in the chain
      * of interceptors in the SDK.
@@ -461,5 +466,23 @@ public String toString() {
           ", expressCreateSession=" + expressCreateSession +
           '}';
     }
+
+    /**
+     * Get the FIPS flag.
+     * @return is fips enabled
+     */
+    public boolean isFipsEnabled() {
+      return fipsEnabled;
+    }
+
+    /**
+     * Set builder value.
+     * @param value new value
+     * @return the builder
+     */
+    public S3ClientCreationParameters withFipsEnabled(final boolean value) {
+      fipsEnabled = value;
+      return this;
+    }
   }
 }

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java

@@ -38,6 +38,7 @@
 import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_STANDARD_OPTIONS;
 import static org.apache.hadoop.fs.s3a.Constants.DIRECTORY_OPERATIONS_PURGE_UPLOADS;
 import static org.apache.hadoop.fs.s3a.Constants.ENABLE_MULTI_DELETE;
+import static org.apache.hadoop.fs.s3a.Constants.FIPS_ENDPOINT;
 import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE;
 import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE_ENABLED;
 import static org.apache.hadoop.fs.s3a.Constants.STORE_CAPABILITY_AWS_V2;
@@ -272,6 +273,7 @@ private InternalConstants() {
           FS_CHECKSUMS,
           FS_MULTIPART_UPLOADER,
           DIRECTORY_LISTING_INCONSISTENT,
+          FIPS_ENDPOINT,
 
           // s3 specific
           STORE_CAPABILITY_AWS_V2,

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java

@@ -357,12 +357,11 @@ public static class BucketInfo extends S3GuardTool {
     public static final String NAME = BUCKET_INFO;
     public static final String GUARDED_FLAG = "guarded";
     public static final String UNGUARDED_FLAG = "unguarded";
-    public static final String AUTH_FLAG = "auth";
-    public static final String NONAUTH_FLAG = "nonauth";
     public static final String ENCRYPTION_FLAG = "encryption";
     public static final String MAGIC_FLAG = "magic";
     public static final String MARKERS_FLAG = "markers";
     public static final String MARKERS_AWARE = "aware";
+    public static final String FIPS_FLAG = "fips";
 
     public static final String PURPOSE = "provide/check information"
         + " about a specific bucket";
@@ -370,8 +369,7 @@ public static class BucketInfo extends S3GuardTool {
     private static final String USAGE = NAME + " [OPTIONS] s3a://BUCKET\n"
         + "\t" + PURPOSE + "\n\n"
         + "Common options:\n"
-        + "  -" + AUTH_FLAG + " - Require the S3Guard mode to be \"authoritative\"\n"
+        + "  -" + FIPS_FLAG + " - Require the client is using a FIPS endpoint\n"
-        + "  -" + NONAUTH_FLAG + " - Require the S3Guard mode to be \"non-authoritative\"\n"
         + "  -" + MAGIC_FLAG +
         " - Require the S3 filesystem to be support the \"magic\" committer\n"
         + "  -" + ENCRYPTION_FLAG
@@ -395,7 +393,7 @@ public static class BucketInfo extends S3GuardTool {
             + " directory markers are not deleted";
 
     public BucketInfo(Configuration conf) {
-      super(conf, GUARDED_FLAG, UNGUARDED_FLAG, AUTH_FLAG, NONAUTH_FLAG, MAGIC_FLAG);
+      super(conf, GUARDED_FLAG, UNGUARDED_FLAG, FIPS_FLAG, MAGIC_FLAG);
       CommandFormat format = getCommandFormat();
       format.addOptionWithValue(ENCRYPTION_FLAG);
       format.addOptionWithValue(MARKERS_FLAG);
@@ -462,6 +460,10 @@ public int run(String[] args, PrintStream out)
       println(out, "\tEndpoint: %s=%s",
           ENDPOINT,
           StringUtils.isNotEmpty(endpoint) ? endpoint : "(unset)");
+      String region = conf.getTrimmed(AWS_REGION, "");
+      println(out, "\tRegion: %s=%s", AWS_REGION,
+          StringUtils.isNotEmpty(region) ? region : "(unset)");
+
       String encryption =
           printOption(out, "\tEncryption", Constants.S3_ENCRYPTION_ALGORITHM,
               "none");
@@ -487,12 +489,12 @@ public int run(String[] args, PrintStream out)
           FS_S3A_COMMITTER_NAME, COMMITTER_NAME_FILE);
       switch (committer) {
       case COMMITTER_NAME_FILE:
-        println(out, "The original 'file' commmitter is active"
+        println(out, "The original 'file' committer is active"
             + " -this is slow and potentially unsafe");
         break;
       case InternalCommitterConstants.COMMITTER_NAME_STAGING:
         println(out, "The 'staging' committer is used "
-            + "-prefer the 'directory' committer");
+            + "-prefer the 'magic' committer");
         // fall through
       case COMMITTER_NAME_DIRECTORY:
         // fall through
@@ -555,13 +557,17 @@ public int run(String[] args, PrintStream out)
       processMarkerOption(out, fs,
           getCommandFormat().getOptValue(MARKERS_FLAG));
 
-      // and check for capabilitities
+      // and check for capabilities
       println(out, "%nStore Capabilities");
       for (String capability : S3A_DYNAMIC_CAPABILITIES) {
         out.printf("\t%s %s%n", capability,
             fs.hasPathCapability(root, capability));
       }
       println(out, "");
+
+      if (commands.getOpt(FIPS_FLAG) && !fs.hasPathCapability(root, FIPS_ENDPOINT)) {
+        throw badState("FIPS endpoint was required but the filesystem is not using it");
+      }
       // and finally flush the output and report a success.
       out.flush();
       return SUCCESS;

hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/connecting.md

@@ -0,0 +1,477 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+# Connecting to an Amazon S3 Bucket through the S3A Connector
+
+<!-- MACRO{toc|fromDepth=0|toDepth=2} -->
+
+
+1. This document covers how to connect to and authenticate with S3 stores, primarily AWS S3.
+2. There have been changes in this mechanism between the V1 and V2 SDK, in particular specifying
+the region is now preferred to specifying the regional S3 endpoint.
+3. For connecting to third-party stores, please read [Working with Third-party S3 Stores](third_party_stores.html) *after* reading this document.
+
+## <a name="foundational"></a> Foundational Concepts
+
+### <a name="regions"></a>  AWS Regions and Availability Zones
+
+AWS provides storage, compute and other services around the world, in *regions*.
+
+Data in S3 is stored *buckets*; each bucket is a single region.
+
+There are some "special" regions: China, AWS GovCloud.
+It is *believed* that the S3A connector works in these places, at least to the extent that nobody has complained about it not working.
+
+### <a name="endpoints"></a> Endpoints
+
+The S3A connector connects to Amazon S3 storage over HTTPS connections, either directly or through an HTTP proxy.
+HTTP HEAD and GET, PUT, POST and DELETE requests are invoked to perform different read/write operations against the store.
+
+There are multiple ways to connect to an S3 bucket
+
+* To an [S3 Endpoint](https://docs.aws.amazon.com/general/latest/gr/s3.html); an HTTPS server hosted by amazon or a third party.
+* To a FIPS-compliant S3 Endpoint.
+* To an AWS S3 [Access Point](https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-points.html).
+* Through a VPC connection, [AWS PrivateLink for Amazon S3](https://docs.aws.amazon.com/AmazonS3/latest/userguide/privatelink-interface-endpoints.html).
+* AWS [Outposts](https://aws.amazon.com/outposts/).
+
+The S3A connector supports all these; S3 Endpoints are the primary mechanism used -either explicitly declared or automatically determined from the declared region of the bucket.
+
+Not supported:
+* AWS [Snowball](https://aws.amazon.com/snowball/).
+
+As of December 2023, AWS S3 uses Transport Layer Security (TLS) [version 1.2](https://aws.amazon.com/blogs/security/tls-1-2-required-for-aws-endpoints/) to secure the communications channel; the S3A client is does this through
+the Apache [HttpClient library](https://hc.apache.org/index.html).
+
+### <a name="third-party"></a> Third party stores
+
+Third-party stores implementing the S3 API are also supported.
+These often only implement a subset of the S3 API; not all features are available.
+If TLS authentication is used, then the HTTPS certificates for the private stores
+_MUST_ be installed on the JVMs on hosts within the Hadoop cluster.
+
+See [Working with Third-party S3 Stores](third_party_stores.html) *after* reading this document.
+
+
+## <a name="settings"></a> Connection Settings
+
+There are three core settings to connect to an S3 store, endpoint, region and whether or not to use path style access.
+
+
+```xml
+<property>
+  <name>fs.s3a.endpoint</name>
+  <description>AWS S3 endpoint to connect to. An up-to-date list is
+    provided in the AWS Documentation: regions and endpoints. Without this
+    property, the standard region (s3.amazonaws.com) is assumed.
+  </description>
+</property>
+
+<property>
+  <name>fs.s3a.endpoint.region</name>
+  <value>REGION</value>
+  <description>AWS Region of the data</description>
+</property>
+
+<property>
+  <name>fs.s3a.path.style.access</name>
+  <value>false</value>
+  <description>Enable S3 path style access by disabling the default virtual hosting behaviour.
+    Needed for AWS PrivateLink, S3 AccessPoints, and, generally, third party stores.
+    Default: false.
+  </description>
+</property>
+```
+
+Historically the S3A connector has preferred the endpoint as defined by the option `fs.s3a.endpoint`.
+With the move to the AWS V2 SDK, there is more emphasis on the region, set by the `fs.s3a.endpoint.region` option.
+
+Normally, declaring the region in `fs.s3a.endpoint.region` should be sufficient to set up the network connection to correctly connect to an AWS-hosted S3 store.
+
+### <a name="timeouts"></a> Network timeouts
+
+See [Timeouts](performance.html#timeouts).
+
+### <a name="networking"></a> Low-level Network Options
+
+```xml
+
+<property>
+  <name>fs.s3a.connection.maximum</name>
+  <value>200</value>
+  <description>Controls the maximum number of simultaneous connections to S3.
+    This must be bigger than the value of fs.s3a.threads.max so as to stop
+    threads being blocked waiting for new HTTPS connections.
+  </description>
+</property>
+
+<property>
+  <name>fs.s3a.connection.ssl.enabled</name>
+  <value>true</value>
+  <description>
+    Enables or disables SSL connections to AWS services.
+  </description>
+</property>
+
+<property>
+  <name>fs.s3a.ssl.channel.mode</name>
+  <value>Default_JSSE</value>
+  <description>
+    TLS implementation and cipher options.
+    Values: OpenSSL, Default, Default_JSSE, Default_JSSE_with_GCM
+
+    Default_JSSE is not truly the the default JSSE implementation because
+    the GCM cipher is disabled when running on Java 8. However, the name
+    was not changed in order to preserve backwards compatibility. Instead,
+    new mode called Default_JSSE_with_GCM delegates to the default JSSE
+    implementation with no changes to the list of enabled ciphers.
+
+    OpenSSL requires the wildfly JAR on the classpath and a compatible installation of the openssl binaries.
+    It is often faster than the JVM libraries, but also trickier to
+    use.
+  </description>
+</property>
+
+<property>
+  <name>fs.s3a.socket.send.buffer</name>
+  <value>8192</value>
+  <description>
+    Socket send buffer hint to amazon connector. Represented in bytes.
+  </description>
+</property>
+
+<property>
+  <name>fs.s3a.socket.recv.buffer</name>
+  <value>8192</value>
+  <description>
+    Socket receive buffer hint to amazon connector. Represented in bytes.
+  </description>
+</property>
+```
+
+### <a name="proxies"></a> Proxy Settings
+
+Connections to S3A stores can be made through an HTTP or HTTPS proxy.
+
+```xml
+<property>
+  <name>fs.s3a.proxy.host</name>
+  <description>
+    Hostname of the (optional) proxy server for S3 connections.
+  </description>
+</property>
+
+<property>
+  <name>fs.s3a.proxy.ssl.enabled</name>
+  <value>false</value>
+  <description>
+    Does the proxy use a TLS connection?
+  </description>
+</property>
+
+<property>
+  <name>fs.s3a.proxy.port</name>
+  <description>
+    Proxy server port. If this property is not set
+    but fs.s3a.proxy.host is, port 80 or 443 is assumed (consistent with
+    the value of fs.s3a.connection.ssl.enabled).
+  </description>
+</property>
+
+<property>
+  <name>fs.s3a.proxy.username</name>
+  <description>Username for authenticating with proxy server.</description>
+</property>
+
+<property>
+  <name>fs.s3a.proxy.password</name>
+  <description>Password for authenticating with proxy server.</description>
+</property>
+
+<property>
+  <name>fs.s3a.proxy.domain</name>
+  <description>Domain for authenticating with proxy server.</description>
+</property>
+
+<property>
+  <name>fs.s3a.proxy.workstation</name>
+  <description>Workstation for authenticating with proxy server.</description>
+</property>
+```
+
+Sometimes the proxy can be source of problems, especially if HTTP connections are kept
+in the connection pool for some time.
+Experiment with the values of `fs.s3a.connection.ttl` and `fs.s3a.connection.request.timeout`
+if long-lived connections have problems.
+
+
+##  <a name="per_bucket_endpoints"></a>Using Per-Bucket Configuration to access data round the world
+
+S3 Buckets are hosted in different "regions", the default being "US-East-1".
+The S3A client talks to this region by default, issuing HTTP requests
+to the server `s3.amazonaws.com`.
+
+S3A can work with buckets from any region. Each region has its own
+S3 endpoint, documented [by Amazon](http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region).
+
+1. Applications running in EC2 infrastructure do not pay for IO to/from
+*local S3 buckets*. They will be billed for access to remote buckets. Always
+use local buckets and local copies of data, wherever possible.
+2. With the V4 signing protocol, AWS requires the explicit region endpoint
+to be used —hence S3A must be configured to use the specific endpoint. This
+is done in the configuration option `fs.s3a.endpoint`.
+3. All endpoints other than the default endpoint only support interaction
+with buckets local to that S3 instance.
+4. Standard S3 buckets support "cross-region" access where use of the original `us-east-1`
+   endpoint allows access to the data, but newer storage types, particularly S3 Express are
+   not supported.
+
+
+
+If the wrong endpoint is used, the request will fail. This may be reported as a 301/redirect error,
+or as a 400 Bad Request: take these as cues to check the endpoint setting of
+a bucket.
+
+The up to date list of regions is [Available online](https://docs.aws.amazon.com/general/latest/gr/s3.html).
+
+This list can be used to specify the endpoint of individual buckets, for example
+for buckets in the central and EU/Ireland endpoints.
+
+```xml
+<property>
+  <name>fs.s3a.bucket.landsat-pds.endpoint</name>
+  <value>s3-us-west-2.amazonaws.com</value>
+</property>
+
+<property>
+  <name>fs.s3a.bucket.eu-dataset.endpoint</name>
+  <value>s3.eu-west-1.amazonaws.com</value>
+</property>
+```
+
+Declaring the region for the data is simpler, as it avoid having to look up the full URL and having to worry about historical quirks of regional endpoint hostnames.
+
+```xml
+<property>
+  <name>fs.s3a.bucket.landsat-pds.endpoint.region</name>
+  <value>us-west-2</value>
+  <description>The endpoint for s3a://landsat-pds URLs</description>
+</property>
+
+<property>
+  <name>fs.s3a.bucket.eu-dataset.endpoint.region</name>
+  <value>eu-west-1</value>
+</property>
+```
+
+
+## <a name="privatelink"></a> AWS PrivateLink
+
+[AWS PrivateLink for Amazon S3](https://docs.aws.amazon.com/AmazonS3/latest/userguide/privatelink-interface-endpoints.html) allows for a private connection to a bucket to be defined, with network access rules managing how a bucket can be accessed.
+
+
+1. Follow the documentation to create the private link
+2. retrieve the DNS name from the console, such as `vpce-f264a96c-6d27bfa7c85e.s3.us-west-2.vpce.amazonaws.com`
+3. Convert this to an endpoint URL by prefixing "https://bucket."
+4. Declare this as the bucket endpoint and switch to path-style access.
+5. Declare the region: there is no automated determination of the region from
+   the `vpce` URL.
+
+```xml
+
+<property>
+  <name>fs.s3a.bucket.example-usw2.endpoint</name>
+  <value>https://bucket.vpce-f264a96c-6d27bfa7c85e.s3.us-west-2.vpce.amazonaws.com/</value>
+</property>
+
+<property>
+  <name>fs.s3a.bucket.example-usw2.path.style.access</name>
+  <value>true</value>
+</property>
+
+<property>
+  <name>fs.s3a.bucket.example-usw2.endpoint.region</name>
+  <value>us-west-2</value>
+</property>
+```
+
+## <a name="fips"></a> Federal Information Processing Standards (FIPS) Endpoints
+
+
+It is possible to use [FIPs-compliant](https://www.nist.gov/itl/fips-general-information) endpoints which
+support a restricted subset of TLS algorithms.
+
+Amazon provide a specific set of [FIPS endpoints](https://aws.amazon.com/compliance/fips/)
+to use so callers can be confident that the network communication is compliant with the standard:
+non-compliant algorithms are unavailable.
+
+The boolean option `fs.s3a.endpoint.fips` (default `false`) switches the S3A connector to using the FIPS endpoint of a region.
+
+```xml
+<property>
+  <name>fs.s3a.endpoint.fips</name>
+  <value>true</value>
+  <description>Use the FIPS endpoint</description>
+</property>
+```
+
+For a single bucket:
+```xml
+<property>
+  <name>fs.s3a.bucket.landsat-pds.endpoint.fips</name>
+  <value>true</value>
+  <description>Use the FIPS endpoint for the landsat dataset</description>
+</property>
+```
+
+If this option is `true`, the endpoint option `fs.s3a.endpoint` MUST NOT be set:
+
+```
+A custom endpoint cannot be combined with FIPS: https://s3.eu-west-2.amazonaws.com
+```
+
+The SDK calculates the FIPS-specific endpoint without any awareness as to whether FIPs is supported by a region. The first attempt to interact with the service will fail
+
+```
+java.net.UnknownHostException: software.amazon.awssdk.core.exception.SdkClientException:
+Received an UnknownHostException when attempting to interact with a service.
+    See cause for the exact endpoint that is failing to resolve.
+    If this is happening on an endpoint that previously worked,
+    there may be a network connectivity issue or your DNS cache
+    could be storing endpoints for too long.:
+    example-london-1.s3-fips.eu-west-2.amazonaws.com
+
+```
+
+*Important* OpenSSL and FIPS endpoints
+
+Linux distributions with an FIPS-compliant SSL library may not be compatible with wildfly.
+Always use with the JDK SSL implementation unless you are confident that the library
+is compatible, or wish to experiment with the settings outside of production deployments.
+
+```xml
+<property>
+  <name>fs.s3a.ssl.channel.mode</name>
+  <value>Default_JSSE</value>
+</property>
+```
+
+## <a name="accesspoints"></a>Configuring S3 AccessPoints usage with S3A
+
+S3A supports [S3 Access Point](https://aws.amazon.com/s3/features/access-points/) usage which
+improves VPC integration with S3 and simplifies your data's permission model because different
+policies can be applied now on the Access Point level. For more information about why to use and
+how to create them make sure to read the official documentation.
+
+Accessing data through an access point, is done by using its ARN, as opposed to just the bucket name.
+You can set the Access Point ARN property using the following per bucket configuration property:
+
+```xml
+<property>
+  <name>fs.s3a.bucket.sample-bucket.accesspoint.arn</name>
+  <value> {ACCESSPOINT_ARN_HERE} </value>
+  <description>Configure S3a traffic to use this AccessPoint</description>
+</property>
+```
+
+This configures access to the `sample-bucket` bucket for S3A, to go through the
+new Access Point ARN. So, for example `s3a://sample-bucket/key` will now use your
+configured ARN when getting data from S3 instead of your bucket.
+
+_the name of the bucket used in the s3a:// URLs is irrelevant; it is not used when connecting with the store_
+
+Example
+
+```xml
+<property>
+  <name>fs.s3a.bucket.example-ap.accesspoint.arn</name>
+  <value>arn:aws:s3:eu-west-2:152813717728:accesspoint/ap-example-london</value>
+  <description>AccessPoint bound to bucket name example-ap</description>
+</property>
+```
+
+The `fs.s3a.accesspoint.required` property can also require all access to S3 to go through Access
+Points. This has the advantage of increasing security inside a VPN / VPC as you only allow access
+to known sources of data defined through Access Points. In case there is a need to access a bucket
+directly (without Access Points) then you can use per bucket overrides to disable this setting on a
+bucket by bucket basis i.e. `fs.s3a.bucket.{YOUR-BUCKET}.accesspoint.required`.
+
+```xml
+<!-- Require access point only access -->
+<property>
+  <name>fs.s3a.accesspoint.required</name>
+  <value>true</value>
+</property>
+<!-- Disable it on a per-bucket basis if needed -->
+<property>
+  <name>fs.s3a.bucket.example-bucket.accesspoint.required</name>
+  <value>false</value>
+</property>
+```
+
+Before using Access Points make sure you're not impacted by the following:
+- `ListObjectsV1` is not supported, this is also deprecated on AWS S3 for performance reasons;
+- The endpoint for S3 requests will automatically change to use
+`s3-accesspoint.REGION.amazonaws.{com | com.cn}` depending on the Access Point ARN. While
+considering endpoints, if you have any custom signers that use the host endpoint property make
+sure to update them if needed;
+
+## <a name="debugging"></a> Debugging network problems
+
+The `storediag` command within the utility [cloudstore](https://github.com/exampleoughran/cloudstore)
+JAR is recommended as the way to view and print settings.
+
+If `storediag` doesn't connect to your S3 store, *nothing else will*.
+
+## <a name="common-problems"></a> Common Sources of Connection Problems
+
+Based on the experience of people who field support calls, here are
+some of the main connectivity issues which cause problems.
+
+### <a name="inconsistent-config"></a> Inconsistent configuration across a cluster
+
+All hosts in the cluster need to have the configuration secrets;
+local environment variables are not enough.
+
+If HTTPS/TLS is used for a private store, the relevant certificates MUST be installed everywhere.
+
+For applications such as distcp, the options need to be passed with the job.
+
+### <a name="public-private-mixup"></a> Confusion between public/private S3 Stores.
+
+If your cluster is configured to use a private store, AWS-hosted buckets are not visible.
+If you wish to read access in a private store, you need to change the endpoint.
+
+Private S3 stores generally expect path style access.
+
+### <a name="region-misconfigure"></a> Region and endpoints misconfigured
+
+These usually surface rapidly and with meaningful messages.
+
+Region errors generally surface as
+* `UnknownHostException`
+* `AWSRedirectException` "Received permanent redirect response to region"
+
+Endpoint configuration problems can be more varied, as they are just HTTPS URLs.
+
+### <a name="wildfly"></a> Wildfly/OpenSSL Brittleness
+
+When it works, it is fast. But it is fussy as to openSSL implementations, TLS protocols and more.
+Because it uses the native openssl binaries, operating system updates can trigger regressions.
+
+Disabling it should be the first step to troubleshooting any TLS problems.
+
+### <a name="proxy-misconfiguration"></a> Proxy setup
+
+If there is a proxy, set it up correctly.

hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md

@@ -33,6 +33,7 @@ full details.
 
 ## <a name="documents"></a> Documents
 
+* [Connecting](./connecting.html)
 * [Encryption](./encryption.html)
 * [Performance](./performance.html)
 * [The upgrade to AWS Java SDK V2](./aws_sdk_upgrade.html)
@@ -223,6 +224,10 @@ Do not inadvertently share these credentials through means such as:
 If you do any of these: change your credentials immediately!
 
 
+## Connecting to Amazon S3 or a third-party store
+
+See [Connecting to an Amazon S3 Bucket through the S3A Connector](connecting.md).
+
 ## <a name="authenticating"></a> Authenticating with S3
 
 Except when interacting with public S3 buckets, the S3A client
@@ -835,61 +840,15 @@ Here are some the S3A properties for use in production.
 </property>
 
 <property>
-  <name>fs.s3a.connection.ssl.enabled</name>
+  <name>fs.s3a.connection.maximum</name>
-  <value>true</value>
+  <value>96</value>
-  <description>Enables or disables SSL connections to AWS services.
+  <description>Controls the maximum number of simultaneous connections to S3.
-    Also sets the default port to use for the s3a proxy settings,
+    This must be bigger than the value of fs.s3a.threads.max so as to stop
-    when not explicitly set in fs.s3a.proxy.port.</description>
+    threads being blocked waiting for new HTTPS connections.
-</property>
+    Why not equal? The AWS SDK transfer manager also uses these connections.
-
-<property>
-  <name>fs.s3a.endpoint</name>
-  <description>AWS S3 endpoint to connect to. An up-to-date list is
-    provided in the AWS Documentation: regions and endpoints. Without this
-    property, the standard region (s3.amazonaws.com) is assumed.
   </description>
 </property>
 
-<property>
-  <name>fs.s3a.path.style.access</name>
-  <value>false</value>
-  <description>Enable S3 path style access ie disabling the default virtual hosting behaviour.
-    Useful for S3A-compliant storage providers as it removes the need to set up DNS for virtual hosting.
-  </description>
-</property>
-
-<property>
-  <name>fs.s3a.proxy.host</name>
-  <description>Hostname of the (optional) proxy server for S3 connections.</description>
-</property>
-
-<property>
-  <name>fs.s3a.proxy.port</name>
-  <description>Proxy server port. If this property is not set
-    but fs.s3a.proxy.host is, port 80 or 443 is assumed (consistent with
-    the value of fs.s3a.connection.ssl.enabled).</description>
-</property>
-
-<property>
-  <name>fs.s3a.proxy.username</name>
-  <description>Username for authenticating with proxy server.</description>
-</property>
-
-<property>
-  <name>fs.s3a.proxy.password</name>
-  <description>Password for authenticating with proxy server.</description>
-</property>
-
-<property>
-  <name>fs.s3a.proxy.domain</name>
-  <description>Domain for authenticating with proxy server.</description>
-</property>
-
-<property>
-  <name>fs.s3a.proxy.workstation</name>
-  <description>Workstation for authenticating with proxy server.</description>
-</property>
-
 <property>
   <name>fs.s3a.attempts.maximum</name>
   <value>5</value>
@@ -1005,14 +964,6 @@ Here are some the S3A properties for use in production.
     implementations can still be used</description>
 </property>
 
-<property>
-  <name>fs.s3a.accesspoint.required</name>
-  <value>false</value>
-  <description>Require that all S3 access is made through Access Points and not through
-  buckets directly. If enabled, use per-bucket overrides to allow bucket access to a specific set
-  of buckets.</description>
-</property>
-
 <property>
   <name>fs.s3a.block.size</name>
   <value>32M</value>
@@ -1218,23 +1169,6 @@ Here are some the S3A properties for use in production.
   </description>
 </property>
 
-<property>
-  <name>fs.s3a.connection.request.timeout</name>
-  <value>0</value>
-  <description>
-    Time out on HTTP requests to the AWS service; 0 means no timeout.
-    Measured in seconds; the usual time suffixes are all supported
-
-    Important: this is the maximum duration of any AWS service call,
-    including upload and copy operations. If non-zero, it must be larger
-    than the time to upload multi-megabyte blocks to S3 from the client,
-    and to rename many-GB files. Use with care.
-
-    Values that are larger than Integer.MAX_VALUE milliseconds are
-    converged to Integer.MAX_VALUE milliseconds
-  </description>
-</property>
-
 <property>
   <name>fs.s3a.etag.checksum.enabled</name>
   <value>false</value>
@@ -1699,179 +1633,6 @@ For a site configuration of:
 The bucket "nightly" will be encrypted with SSE-KMS using the KMS key
 `arn:aws:kms:eu-west-2:1528130000000:key/753778e4-2d0f-42e6-b894-6a3ae4ea4e5f`
 
-###  <a name="per_bucket_endpoints"></a>Using Per-Bucket Configuration to access data round the world
-
-S3 Buckets are hosted in different "regions", the default being "US-East".
-The S3A client talks to this region by default, issuing HTTP requests
-to the server `s3.amazonaws.com`.
-
-S3A can work with buckets from any region. Each region has its own
-S3 endpoint, documented [by Amazon](http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region).
-
-1. Applications running in EC2 infrastructure do not pay for IO to/from
-*local S3 buckets*. They will be billed for access to remote buckets. Always
-use local buckets and local copies of data, wherever possible.
-1. The default S3 endpoint can support data IO with any bucket when the V1 request
-signing protocol is used.
-1. When the V4 signing protocol is used, AWS requires the explicit region endpoint
-to be used —hence S3A must be configured to use the specific endpoint. This
-is done in the configuration option `fs.s3a.endpoint`.
-1. All endpoints other than the default endpoint only support interaction
-with buckets local to that S3 instance.
-
-While it is generally simpler to use the default endpoint, working with
-V4-signing-only regions (Frankfurt, Seoul) requires the endpoint to be identified.
-Expect better performance from direct connections —traceroute will give you some insight.
-
-If the wrong endpoint is used, the request may fail. This may be reported as a 301/redirect error,
-or as a 400 Bad Request: take these as cues to check the endpoint setting of
-a bucket.
-
-Here is a list of properties defining all AWS S3 regions, current as of June 2017:
-
-```xml
-<!--
- This is the default endpoint, which can be used to interact
- with any v2 region.
- -->
-<property>
-  <name>central.endpoint</name>
-  <value>s3.amazonaws.com</value>
-</property>
-
-<property>
-  <name>canada.endpoint</name>
-  <value>s3.ca-central-1.amazonaws.com</value>
-</property>
-
-<property>
-  <name>frankfurt.endpoint</name>
-  <value>s3.eu-central-1.amazonaws.com</value>
-</property>
-
-<property>
-  <name>ireland.endpoint</name>
-  <value>s3-eu-west-1.amazonaws.com</value>
-</property>
-
-<property>
-  <name>london.endpoint</name>
-  <value>s3.eu-west-2.amazonaws.com</value>
-</property>
-
-<property>
-  <name>mumbai.endpoint</name>
-  <value>s3.ap-south-1.amazonaws.com</value>
-</property>
-
-<property>
-  <name>ohio.endpoint</name>
-  <value>s3.us-east-2.amazonaws.com</value>
-</property>
-
-<property>
-  <name>oregon.endpoint</name>
-  <value>s3-us-west-2.amazonaws.com</value>
-</property>
-
-<property>
-  <name>sao-paolo.endpoint</name>
-  <value>s3-sa-east-1.amazonaws.com</value>
-</property>
-
-<property>
-  <name>seoul.endpoint</name>
-  <value>s3.ap-northeast-2.amazonaws.com</value>
-</property>
-
-<property>
-  <name>singapore.endpoint</name>
-  <value>s3-ap-southeast-1.amazonaws.com</value>
-</property>
-
-<property>
-  <name>sydney.endpoint</name>
-  <value>s3-ap-southeast-2.amazonaws.com</value>
-</property>
-
-<property>
-  <name>tokyo.endpoint</name>
-  <value>s3-ap-northeast-1.amazonaws.com</value>
-</property>
-
-<property>
-  <name>virginia.endpoint</name>
-  <value>${central.endpoint}</value>
-</property>
-```
-
-This list can be used to specify the endpoint of individual buckets, for example
-for buckets in the central and EU/Ireland endpoints.
-
-```xml
-<property>
-  <name>fs.s3a.bucket.landsat-pds.endpoint</name>
-  <value>${central.endpoint}</value>
-  <description>The endpoint for s3a://landsat-pds URLs</description>
-</property>
-
-<property>
-  <name>fs.s3a.bucket.eu-dataset.endpoint</name>
-  <value>${ireland.endpoint}</value>
-  <description>The endpoint for s3a://eu-dataset URLs</description>
-</property>
-```
-
-Why explicitly declare a bucket bound to the central endpoint? It ensures
-that if the default endpoint is changed to a new region, data store in
-US-east is still reachable.
-
-## <a name="accesspoints"></a>Configuring S3 AccessPoints usage with S3A
-S3a now supports [S3 Access Point](https://aws.amazon.com/s3/features/access-points/) usage which
-improves VPC integration with S3 and simplifies your data's permission model because different
-policies can be applied now on the Access Point level. For more information about why to use and
-how to create them make sure to read the official documentation.
-
-Accessing data through an access point, is done by using its ARN, as opposed to just the bucket name.
-You can set the Access Point ARN property using the following per bucket configuration property:
-```xml
-<property>
-    <name>fs.s3a.bucket.sample-bucket.accesspoint.arn</name>
-    <value> {ACCESSPOINT_ARN_HERE} </value>
-    <description>Configure S3a traffic to use this AccessPoint</description>
-</property>
-```
-
-This configures access to the `sample-bucket` bucket for S3A, to go through the
-new Access Point ARN. So, for example `s3a://sample-bucket/key` will now use your
-configured ARN when getting data from S3 instead of your bucket.
-
-The `fs.s3a.accesspoint.required` property can also require all access to S3 to go through Access
-Points. This has the advantage of increasing security inside a VPN / VPC as you only allow access
-to known sources of data defined through Access Points. In case there is a need to access a bucket
-directly (without Access Points) then you can use per bucket overrides to disable this setting on a
-bucket by bucket basis i.e. `fs.s3a.bucket.{YOUR-BUCKET}.accesspoint.required`.
-
-```xml
-<!-- Require access point only access -->
-<property>
-    <name>fs.s3a.accesspoint.required</name>
-    <value>true</value>
-</property>
-<!-- Disable it on a per-bucket basis if needed -->
-<property>
-    <name>fs.s3a.bucket.example-bucket.accesspoint.required</name>
-    <value>false</value>
-</property>
-```
-
-Before using Access Points make sure you're not impacted by the following:
-- `ListObjectsV1` is not supported, this is also deprecated on AWS S3 for performance reasons;
-- The endpoint for S3 requests will automatically change from `s3.amazonaws.com` to use
-`s3-accesspoint.REGION.amazonaws.{com | com.cn}` depending on the Access Point ARN. While
-considering endpoints, if you have any custom signers that use the host endpoint property make
-sure to update them if needed;
-
 ## <a name="requester_pays"></a>Requester Pays buckets
 
 S3A supports buckets with

hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md

@@ -218,6 +218,7 @@ everything uses the same HTTP connection pool.
 | `fs.s3a.executor.capacity`     | `16`    | Maximum threads for any single operation                         |
 | `fs.s3a.max.total.tasks`       | `16`    | Extra tasks which can be queued excluding prefetching operations |
 
+### <a name="timeouts"></a> Timeouts.
 
 Network timeout options can be tuned to make the client fail faster *or* retry more.
 The choice is yours. Generally recovery is better, but sometimes fail-fast is more useful.

hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md

@@ -132,20 +132,17 @@ This auditing information can be used to identify opportunities to reduce load.
 Prints and optionally checks the status of a bucket.
 
 ```bash
-hadoop s3guard bucket-info [-guarded] [-unguarded] [-auth] [-nonauth] [-magic] [-encryption ENCRYPTION] [-markers MARKER] s3a://BUCKET
+hadoop s3guard bucket-info [-fips] [-magic] [-encryption ENCRYPTION] [-markers MARKER] s3a://BUCKET
 ```
 
 Options
 
-| argument | meaning |
+| argument             | meaning                                                             |
-|-----------|-------------|
+|----------------------|---------------------------------------------------------------------|
-| `-guarded` | Require S3Guard to be enabled. This will now always fail |
+| `-fips`              | Require FIPS endopint to be in use                                  |
-| `-unguarded` | Require S3Guard to be disabled. This will now always succeed |
+| `-magic`             | Require the S3 filesystem to be support the "magic" committer       |
-| `-auth` | Require the S3Guard mode to be "authoritative". This will now always fail |
+| `-markers`           | Directory marker status: `aware`, `keep`, `delete`, `authoritative` |
-| `-nonauth` | Require the S3Guard mode to be "non-authoritative". This will now always fail |
+| `-encryption <type>` | Require a specific encryption algorithm                             |
-| `-magic` | Require the S3 filesystem to be support the "magic" committer |
-| `-markers` | Directory marker status: `aware`, `keep`, `delete`, `authoritative` |
-| `-encryption <type>` | Require a specific encryption algorithm  |
 
 The server side encryption options are not directly related to S3Guard, but
 it is often convenient to check them at the same time.

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABucketExistence.java

@@ -39,6 +39,7 @@
 import static org.apache.hadoop.fs.s3a.Constants.AWS_REGION;
 import static org.apache.hadoop.fs.s3a.Constants.AWS_S3_ACCESSPOINT_REQUIRED;
 import static org.apache.hadoop.fs.s3a.Constants.ENDPOINT;
+import static org.apache.hadoop.fs.s3a.Constants.FIPS_ENDPOINT;
 import static org.apache.hadoop.fs.s3a.Constants.FS_S3A;
 import static org.apache.hadoop.fs.s3a.Constants.PATH_STYLE_ACCESS;
 import static org.apache.hadoop.fs.s3a.Constants.S3A_BUCKET_PROBE;
@@ -138,6 +139,7 @@ private Configuration createConfigurationWithProbe(final int probe) {
     removeBaseAndBucketOverrides(conf,
         S3A_BUCKET_PROBE,
         ENDPOINT,
+        FIPS_ENDPOINT,
         AWS_REGION,
         PATH_STYLE_ACCESS);
     conf.setInt(S3A_BUCKET_PROBE, probe);

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java

@@ -20,7 +20,6 @@
 
 import java.io.IOException;
 import java.net.URI;
-import java.net.URISyntaxException;
 import java.net.UnknownHostException;
 import java.nio.file.AccessDeniedException;
 import java.util.ArrayList;
@@ -36,16 +35,17 @@
 import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
 import software.amazon.awssdk.services.s3.S3Client;
 import software.amazon.awssdk.services.s3.model.HeadBucketRequest;
+import software.amazon.awssdk.services.s3.model.HeadBucketResponse;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext;
 
 import static org.apache.hadoop.fs.s3a.Constants.AWS_REGION;
+import static org.apache.hadoop.fs.s3a.Constants.CENTRAL_ENDPOINT;
 import static org.apache.hadoop.fs.s3a.Constants.PATH_STYLE_ACCESS;
+import static org.apache.hadoop.fs.s3a.DefaultS3ClientFactory.ERROR_ENDPOINT_WITH_FIPS;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
 import static org.apache.hadoop.io.IOUtils.closeStream;
-import static org.apache.hadoop.fs.s3a.Constants.CENTRAL_ENDPOINT;
-
 import static org.apache.hadoop.test.LambdaTestUtils.intercept;
 
 /**
@@ -82,6 +82,8 @@ public class ITestS3AEndpointRegion extends AbstractS3ATestBase {
 
   private static final String VPC_ENDPOINT = "vpce-1a2b3c4d-5e6f.s3.us-west-2.vpce.amazonaws.com";
 
+  public static final String EXCEPTION_THROWN_BY_INTERCEPTOR = "Exception thrown by interceptor";
+
   /**
    * New FS instance which will be closed in teardown.
    */
@@ -134,10 +136,9 @@ public void testEndpointOverride() throws Throwable {
     describe("Create a client with a configured endpoint");
     Configuration conf = getConfiguration();
 
-    S3Client client = createS3Client(conf, AWS_ENDPOINT_TEST, null, US_EAST_2);
+    S3Client client = createS3Client(conf, AWS_ENDPOINT_TEST, null, US_EAST_2, false);
 
-    intercept(AwsServiceException.class, "Exception thrown by interceptor", () -> client.headBucket(
+    expectInterceptorException(client);
-        HeadBucketRequest.builder().bucket(getFileSystem().getBucket()).build()));
   }
 
   @Test
@@ -145,10 +146,9 @@ public void testCentralEndpoint() throws Throwable {
     describe("Create a client with the central endpoint");
     Configuration conf = getConfiguration();
 
-    S3Client client = createS3Client(conf, CENTRAL_ENDPOINT, null, US_EAST_1);
+    S3Client client = createS3Client(conf, CENTRAL_ENDPOINT, null, US_EAST_1, false);
 
-    intercept(AwsServiceException.class, "Exception thrown by interceptor", () -> client.headBucket(
+    expectInterceptorException(client);
-        HeadBucketRequest.builder().bucket(getFileSystem().getBucket()).build()));
   }
 
   @Test
@@ -156,21 +156,40 @@ public void testWithRegionConfig() throws Throwable {
     describe("Create a client with a configured region");
     Configuration conf = getConfiguration();
 
-    S3Client client = createS3Client(conf, null, EU_WEST_2, EU_WEST_2);
+    S3Client client = createS3Client(conf, null, EU_WEST_2, EU_WEST_2, false);
 
-    intercept(AwsServiceException.class, "Exception thrown by interceptor", () -> client.headBucket(
+    expectInterceptorException(client);
-        HeadBucketRequest.builder().bucket(getFileSystem().getBucket()).build()));
   }
 
+  @Test
+  public void testWithFips() throws Throwable {
+    describe("Create a client with fips enabled");
 
+    S3Client client = createS3Client(getConfiguration(),
+        null, EU_WEST_2, EU_WEST_2, true);
+    expectInterceptorException(client);
+  }
+
+  /**
+   * Attempting to create a client with fips enabled and an endpoint specified
+   * fails during client construction.
+   */
+  @Test
+  public void testWithFipsAndEndpoint() throws Throwable {
+    describe("Create a client with fips and an endpoint");
+
+    intercept(IllegalArgumentException.class, ERROR_ENDPOINT_WITH_FIPS, () ->
+        createS3Client(getConfiguration(), CENTRAL_ENDPOINT, null, US_EAST_1, true));
+  }
+
+  @Test
   public void testEUWest2Endpoint() throws Throwable {
     describe("Create a client with the eu west 2 endpoint");
     Configuration conf = getConfiguration();
 
-    S3Client client = createS3Client(conf, EU_WEST_2_ENDPOINT, null, EU_WEST_2);
+    S3Client client = createS3Client(conf, EU_WEST_2_ENDPOINT, null, EU_WEST_2, false);
 
-    intercept(AwsServiceException.class, "Exception thrown by interceptor", () -> client.headBucket(
+    expectInterceptorException(client);
-        HeadBucketRequest.builder().bucket(getFileSystem().getBucket()).build()));
   }
 
   @Test
@@ -178,10 +197,9 @@ public void testWithRegionAndEndpointConfig() throws Throwable {
     describe("Test that when both region and endpoint are configured, region takes precedence");
     Configuration conf = getConfiguration();
 
-    S3Client client = createS3Client(conf, EU_WEST_2_ENDPOINT, US_WEST_2, US_WEST_2);
+    S3Client client = createS3Client(conf, EU_WEST_2_ENDPOINT, US_WEST_2, US_WEST_2, false);
 
-    intercept(AwsServiceException.class, "Exception thrown by interceptor", () -> client.headBucket(
+    expectInterceptorException(client);
-        HeadBucketRequest.builder().bucket(getFileSystem().getBucket()).build()));
   }
 
   @Test
@@ -189,21 +207,43 @@ public void testWithChinaEndpoint() throws Throwable {
     describe("Test with a china endpoint");
     Configuration conf = getConfiguration();
 
-    S3Client client = createS3Client(conf, CN_ENDPOINT, null, CN_NORTHWEST_1);
+    S3Client client = createS3Client(conf, CN_ENDPOINT, null, CN_NORTHWEST_1, false);
 
-    intercept(AwsServiceException.class, "Exception thrown by interceptor", () -> client.headBucket(
+    expectInterceptorException(client);
-        HeadBucketRequest.builder().bucket(getFileSystem().getBucket()).build()));
+  }
+
+  /**
+   * Expect an exception to be thrown by the interceptor with the message
+   * {@link #EXCEPTION_THROWN_BY_INTERCEPTOR}.
+   * @param client client to issue a head request against.
+   * @return the expected exception.
+   * @throws Exception any other exception.
+   */
+  private AwsServiceException expectInterceptorException(final S3Client client)
+      throws Exception {
+
+    return intercept(AwsServiceException.class, EXCEPTION_THROWN_BY_INTERCEPTOR,
+        () -> head(client));
+  }
+
+  /**
+   * Issue a head request against the bucket.
+   * @param client client to use
+   * @return the response.
+   */
+  private HeadBucketResponse head(final S3Client client) {
+    return client.headBucket(
+        HeadBucketRequest.builder().bucket(getFileSystem().getBucket()).build());
   }
 
   @Test
   public void testWithGovCloudEndpoint() throws Throwable {
-    describe("Test with a gov cloud endpoint");
+    describe("Test with a gov cloud endpoint; enable fips");
     Configuration conf = getConfiguration();
 
-    S3Client client = createS3Client(conf, GOV_ENDPOINT, null, US_GOV_EAST_1);
+    S3Client client = createS3Client(conf, GOV_ENDPOINT, null, US_GOV_EAST_1, false);
 
-    intercept(AwsServiceException.class, "Exception thrown by interceptor", () -> client.headBucket(
+    expectInterceptorException(client);
-        HeadBucketRequest.builder().bucket(getFileSystem().getBucket()).build()));
   }
 
   @Test
@@ -212,19 +252,20 @@ public void testWithVPCE() throws Throwable {
     describe("Test with vpc endpoint");
     Configuration conf = getConfiguration();
 
-    S3Client client = createS3Client(conf, VPC_ENDPOINT, null, US_WEST_2);
+    S3Client client = createS3Client(conf, VPC_ENDPOINT, null, US_WEST_2, false);
 
-    intercept(AwsServiceException.class, "Exception thrown by interceptor", () -> client.headBucket(
+    expectInterceptorException(client);
-        HeadBucketRequest.builder().bucket(getFileSystem().getBucket()).build()));
   }
 
-  class RegionInterceptor implements ExecutionInterceptor {
+  private final class RegionInterceptor implements ExecutionInterceptor {
-    private String endpoint;
+    private final String endpoint;
-    private String region;
+    private final String region;
+    private final boolean isFips;
 
-    RegionInterceptor(String endpoint, String region) {
+    RegionInterceptor(String endpoint, String region, final boolean isFips) {
       this.endpoint = endpoint;
       this.region = region;
+      this.isFips = isFips;
     }
 
     @Override
@@ -249,8 +290,15 @@ public void beforeExecution(Context.BeforeExecution context,
               executionAttributes.getAttribute(AwsExecutionAttribute.AWS_REGION).toString())
           .describedAs("Incorrect region set").isEqualTo(region);
 
+      // verify the fips state matches expectation.
+      Assertions.assertThat(executionAttributes.getAttribute(
+          AwsExecutionAttribute.FIPS_ENDPOINT_ENABLED))
+          .describedAs("Incorrect FIPS flag set in execution attributes")
+          .isNotNull()
+          .isEqualTo(isFips);
+
       // We don't actually want to make a request, so exit early.
-      throw AwsServiceException.builder().message("Exception thrown by interceptor").build();
+      throw AwsServiceException.builder().message(EXCEPTION_THROWN_BY_INTERCEPTOR).build();
     }
   }
 
@@ -261,17 +309,17 @@ public void beforeExecution(Context.BeforeExecution context,
    * @param conf configuration to use.
    * @param endpoint endpoint.
    * @param expectedRegion the region that should be set in the client.
+   * @param isFips is this a FIPS endpoint?
    * @return the client.
-   * @throws URISyntaxException parse problems.
    * @throws IOException IO problems
    */
   @SuppressWarnings("deprecation")
   private S3Client createS3Client(Configuration conf,
-      String endpoint, String configuredRegion, String expectedRegion)
+      String endpoint, String configuredRegion, String expectedRegion, boolean isFips)
       throws IOException {
 
     List<ExecutionInterceptor> interceptors = new ArrayList<>();
-    interceptors.add(new RegionInterceptor(endpoint, expectedRegion));
+    interceptors.add(new RegionInterceptor(endpoint, expectedRegion, isFips));
 
     DefaultS3ClientFactory factory
         = new DefaultS3ClientFactory();
@@ -283,8 +331,8 @@ private S3Client createS3Client(Configuration conf,
         .withMetrics(new EmptyS3AStatisticsContext()
             .newStatisticsFromAwsSdk())
         .withExecutionInterceptors(interceptors)
-        .withRegion(configuredRegion);
+        .withRegion(configuredRegion)
-
+        .withFipsEnabled(isFips);
 
     S3Client client = factory.createS3Client(
         getFileSystem().getUri(),

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardTool.java

@@ -33,6 +33,8 @@
 import org.apache.hadoop.test.LambdaTestUtils;
 import org.apache.hadoop.util.StringUtils;
 
+import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
+import static org.apache.hadoop.fs.s3a.Constants.FIPS_ENDPOINT;
 import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_ALGORITHM;
 import static org.apache.hadoop.fs.s3a.MultipartTestUtils.assertNoUploadsAt;
 import static org.apache.hadoop.fs.s3a.MultipartTestUtils.clearAnyUploads;
@@ -97,6 +99,20 @@ public void testStoreInfo() throws Throwable {
     LOG.info("Exec output=\n{}", output);
   }
 
+  @Test
+  public void testStoreInfoFips() throws Throwable {
+    final S3AFileSystem fs = getFileSystem();
+    if (!fs.hasPathCapability(new Path("/"), FIPS_ENDPOINT)) {
+      skip("FIPS not enabled");
+    }
+    S3GuardTool.BucketInfo cmd =
+        toClose(new S3GuardTool.BucketInfo(fs.getConf()));
+    String output = exec(cmd, cmd.getName(),
+        "-" + BucketInfo.FIPS_FLAG,
+        fs.getUri().toString());
+    LOG.info("Exec output=\n{}", output);
+  }
+
   private final static String UPLOAD_NAME = "test-upload";
 
   @Test

hadoop-tools/hadoop-aws/src/test/resources/core-site.xml

@@ -56,6 +56,12 @@
     <description>Do not add the referrer header to landsat operations</description>
   </property>
 
+  <property>
+    <name>fs.s3a.bucket.landsat-pds.endpoint.fips</name>
+    <value>true</value>
+    <description>Use the fips endpoint</description>
+  </property>
+
   <!-- Per-bucket configurations: usgs-landsat -->
   <property>
     <name>fs.s3a.bucket.usgs-landsat.endpoint.region</name>