From 3f98cb674168085dfb28df0cc22a036b9e788a03 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Wed, 7 Feb 2024 12:07:54 +0000 Subject: [PATCH] HADOOP-19045. CreateSession Timeout - followup (#6532) This is a followup to PR: HADOOP-19045. S3A: Validate CreateSession Timeout Propagation (#6470) Remove all declarations of fs.s3a.connection.request.timeout in - hadoop-common/src/main/resources/core-default.xml - hadoop-aws/src/test/resources/core-site.xml New test in TestAwsClientConfig to verify that the value defined in fs.s3a.Constants class is used. This is brittle to someone overriding it in their test setups, but as this test is intended to verify that the option is not explicitly set, there's no workaround. Contributed by Steve Loughran --- .../src/main/resources/core-default.xml | 14 -------------- .../markdown/tools/hadoop-aws/performance.md | 2 +- .../tools/hadoop-aws/troubleshooting_s3a.md | 2 +- .../hadoop/fs/s3a/impl/TestAwsClientConfig.java | 16 ++++++++++++++++ .../hadoop-aws/src/test/resources/core-site.xml | 5 ----- 5 files changed, 18 insertions(+), 21 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 29ec06db65..00533e362b 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -2097,20 +2097,6 @@ - - fs.s3a.connection.request.timeout - 0s - - Time out on HTTP requests to the AWS service; 0 means no timeout. - - Important: this is the maximum duration of any AWS service call, - including upload and copy operations. If non-zero, it must be larger - than the time to upload multi-megabyte blocks to S3 from the client, - and to rename many-GB files. Use with care. - - - - fs.s3a.etag.checksum.enabled false diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md index 4c03cca171..44c8086d4f 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md @@ -231,7 +231,7 @@ The choice is yours. Generally recovery is better, but sometimes fail-fast is mo | `fs.s3a.connection.acquisition.timeout` | `60s` | `*` | Timeout for waiting for a connection from the pool. | | `fs.s3a.connection.establish.timeout` | `30s` | | Time to establish the TCP/TLS connection | | `fs.s3a.connection.idle.time` | `60s` | `*` | Maximum time for idle HTTP connections in the pool | -| `fs.s3a.connection.request.timeout` | `0` | | If greater than zero, maximum duration of any request | +| `fs.s3a.connection.request.timeout` | `60s` | | If greater than zero, maximum time for a response | | `fs.s3a.connection.timeout` | `200s` | | Timeout for socket problems on a TCP channel | | `fs.s3a.connection.ttl` | `5m` | | Lifetime of HTTP connections from the pool | diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md index c1b499e3da..e53e4a0022 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md @@ -1510,7 +1510,7 @@ It is possible to configure a global timeout for AWS service calls using followi ```xml fs.s3a.connection.request.timeout - 0 + 5m Time out on HTTP requests to the AWS service; 0 means no timeout. Measured in seconds; the usual time suffixes are all supported diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestAwsClientConfig.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestAwsClientConfig.java index 2a9e05755c..eacff90ea4 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestAwsClientConfig.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestAwsClientConfig.java @@ -40,6 +40,7 @@ import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_CONNECTION_TTL_DURATION; import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_ESTABLISH_TIMEOUT_DURATION; import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_MAXIMUM_CONNECTIONS; +import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_REQUEST_TIMEOUT_DURATION; import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_SOCKET_TIMEOUT_DURATION; import static org.apache.hadoop.fs.s3a.Constants.ESTABLISH_TIMEOUT; import static org.apache.hadoop.fs.s3a.Constants.MAXIMUM_CONNECTIONS; @@ -175,6 +176,21 @@ public void testCreateApiConnectionSettings() { .describedAs("%s in %s", REQUEST_TIMEOUT, settings) .isEqualTo(Duration.ofHours(1)); } + /** + * Verify that the timeout from {@link org.apache.hadoop.fs.s3a.Constants#DEFAULT_REQUEST_TIMEOUT_DURATION} + * makes it all the way through and that nothing in in core-default or core-site is setting it. + * This test will fail if someone does set it in core-site.xml + */ + @Test + public void testCreateApiConnectionSettingsDefault() { + final Configuration conf = new Configuration(); + Assertions.assertThat(conf.get(REQUEST_TIMEOUT)) + .describedAs("Request timeout %s", REQUEST_TIMEOUT) + .isNull(); + + assertDuration(REQUEST_TIMEOUT, DEFAULT_REQUEST_TIMEOUT_DURATION, + createApiConnectionSettings(conf).getApiCallTimeout()); + } /** * Set a list of keys to the same value. diff --git a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml index 70b87ee275..30e54ca0b8 100644 --- a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml +++ b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml @@ -201,11 +201,6 @@ true - - fs.s3a.connection.request.timeout - 10s - - fs.s3a.attempts.maximum 1