big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-11521. Make connection timeout configurable in s3a. (Thomas Demoor via stevel)

Browse Source
This commit is contained in:
Steve Loughran 2015-02-17 20:00:00 +00:00
parent 6be5670949
commit 00b80958d8
5 changed files with 21 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -615,6 +615,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-11522. Update S3A Documentation. (Thomas Demoor via stevel) HADOOP-11522. Update S3A Documentation. (Thomas Demoor via stevel)
HADOOP-11521. Make connection timeout configurable in s3a.
(Thomas Demoor via stevel)
OPTIMIZATIONS OPTIMIZATIONS
HADOOP-11323. WritableComparator#compare keeps reference to byte array. HADOOP-11323. WritableComparator#compare keeps reference to byte array.

6
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
View File

@ -757,6 +757,12 @@ for ldap providers in the same way as above does.
<description>How many times we should retry commands on transient errors.</description> <description>How many times we should retry commands on transient errors.</description>
</property> </property>
<property>
<name>fs.s3a.connection.establish.timeout</name>
<value>5000</value>
<description>Socket connection setup timeout in seconds.</description>
</property>
<property> <property>
<name>fs.s3a.connection.timeout</name> <name>fs.s3a.connection.timeout</name>
<value>50000</value> <value>50000</value>

4
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
View File

@ -42,6 +42,10 @@ public class Constants {
// number of times we should retry errors // number of times we should retry errors
public static final String MAX_ERROR_RETRIES = "fs.s3a.attempts.maximum"; public static final String MAX_ERROR_RETRIES = "fs.s3a.attempts.maximum";
public static final int DEFAULT_MAX_ERROR_RETRIES = 10; public static final int DEFAULT_MAX_ERROR_RETRIES = 10;
// seconds until we give up trying to establish a connection to s3
public static final String ESTABLISH_TIMEOUT = "fs.s3a.connection.establish.timeout";
public static final int DEFAULT_ESTABLISH_TIMEOUT = 50000;
// seconds until we give up on a connection to s3 // seconds until we give up on a connection to s3
public static final String SOCKET_TIMEOUT = "fs.s3a.connection.timeout"; public static final String SOCKET_TIMEOUT = "fs.s3a.connection.timeout";

2
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
View File

@ -174,6 +174,8 @@ public void initialize(URI name, Configuration conf) throws IOException {
awsConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP); awsConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP);
awsConf.setMaxErrorRetry(conf.getInt(MAX_ERROR_RETRIES, awsConf.setMaxErrorRetry(conf.getInt(MAX_ERROR_RETRIES,
DEFAULT_MAX_ERROR_RETRIES)); DEFAULT_MAX_ERROR_RETRIES));
awsConf.setConnectionTimeout(conf.getInt(ESTABLISH_TIMEOUT,
DEFAULT_ESTABLISH_TIMEOUT));
awsConf.setSocketTimeout(conf.getInt(SOCKET_TIMEOUT, awsConf.setSocketTimeout(conf.getInt(SOCKET_TIMEOUT,
DEFAULT_SOCKET_TIMEOUT)); DEFAULT_SOCKET_TIMEOUT));

6
hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
View File

@ -210,6 +210,12 @@ If you do any of these: change your credentials immediately!
<description>How many times we should retry commands on transient errors.</description> <description>How many times we should retry commands on transient errors.</description>
</property> </property>
<property>
<name>fs.s3a.connection.establish.timeout</name>
<value>5000</value>
<description>Socket connection setup timeout in seconds.</description>
</property>
<property> <property>
<name>fs.s3a.connection.timeout</name> <name>fs.s3a.connection.timeout</name>
<value>50000</value> <value>50000</value>