From e28f83a1eb3eb0a9004d91fff18c238590da1d92 Mon Sep 17 00:00:00 2001 From: Mukund Thakur Date: Tue, 19 Dec 2023 14:04:07 -0600 Subject: [PATCH] HADOOP-19015. Increase fs.s3a.connection.maximum to 500 to minimize risk of Timeout waiting for connection from pool --- .../hadoop-common/src/main/resources/core-default.xml | 2 +- .../src/main/java/org/apache/hadoop/fs/s3a/Constants.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 5a5171056d..6ff14b2d61 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1530,7 +1530,7 @@ fs.s3a.connection.maximum - 200 + 500 Controls the maximum number of simultaneous connections to S3. This must be bigger than the value of fs.s3a.threads.max so as to stop threads being blocked waiting for new HTTPS connections. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index fb4f22cedb..05e6a43a05 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -173,7 +173,7 @@ private Constants() { * Future releases are likely to increase this value. * Keep in sync with the value in {@code core-default.xml} */ - public static final int DEFAULT_MAXIMUM_CONNECTIONS = 200; + public static final int DEFAULT_MAXIMUM_CONNECTIONS = 500; /** * Configuration option to configure expiration time of