big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-13212 Provide an option to set the socket buffers in S3AFileSystem (Rajesh Balamohan)

Browse Source
This commit is contained in:
Steve Loughran 2016-07-20 13:42:19 +01:00
parent 9ccf935b2c
commit 37362c2f92
5 changed files with 40 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
View File

@ -907,6 +907,18 @@
<description>Socket connection timeout in milliseconds.</description>
</property>
<property>
<name>fs.s3a.socket.send.buffer</name>
<value>8192</value>
<description>Socket send buffer hint to amazon connector. Represented in bytes.</description>
</property>
<property>
<name>fs.s3a.socket.recv.buffer</name>
<value>8192</value>
<description>Socket receive buffer hint to amazon connector. Represented in bytes.</description>
</property>
<property>
<name>fs.s3a.paging.maximum</name>
<value>5000</value>

8
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
View File

@ -82,6 +82,14 @@ private Constants() {
public static final String SOCKET_TIMEOUT = "fs.s3a.connection.timeout";
public static final int DEFAULT_SOCKET_TIMEOUT = 200000;
// socket send buffer to be used in Amazon client
public static final String SOCKET_SEND_BUFFER = "fs.s3a.socket.send.buffer";
public static final int DEFAULT_SOCKET_SEND_BUFFER = 8 * 1024;
// socket send buffer to be used in Amazon client
public static final String SOCKET_RECV_BUFFER = "fs.s3a.socket.recv.buffer";
public static final int DEFAULT_SOCKET_RECV_BUFFER = 8 * 1024;
// number of records to get while paging through a directory listing
public static final String MAX_PAGING_KEYS = "fs.s3a.paging.maximum";
public static final int DEFAULT_MAX_PAGING_KEYS = 5000;

5
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
View File

@ -163,6 +163,11 @@ public void initialize(URI name, Configuration conf) throws IOException {
DEFAULT_ESTABLISH_TIMEOUT, 0));
awsConf.setSocketTimeout(intOption(conf, SOCKET_TIMEOUT,
DEFAULT_SOCKET_TIMEOUT, 0));
int sockSendBuffer = intOption(conf, SOCKET_SEND_BUFFER,
DEFAULT_SOCKET_SEND_BUFFER, 2048);
int sockRecvBuffer = intOption(conf, SOCKET_RECV_BUFFER,
DEFAULT_SOCKET_RECV_BUFFER, 2048);
awsConf.setSocketBufferSizeHints(sockSendBuffer, sockRecvBuffer);
String signerOverride = conf.getTrimmed(SIGNING_ALGORITHM, "");
if (!signerOverride.isEmpty()) {
LOG.debug("Signer override = {}", signerOverride);

12
hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
View File

@ -503,6 +503,18 @@ this capability.
which each use a thread from the threadpool.</description>
</property>
<property>
<name>fs.s3a.socket.send.buffer</name>
<value>8192</value>
<description>Socket send buffer hint to amazon connector. Represented in bytes.</description>
</property>
<property>
<name>fs.s3a.socket.recv.buffer</name>
<value>8192</value>
<description>Socket receive buffer hint to amazon connector. Represented in bytes.</description>
</property>
<property>
<name>fs.s3a.threads.keepalivetime</name>
<value>60</value>

3
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3AInputStreamPerformance.java
View File

@ -44,6 +44,7 @@
import java.io.IOException;
import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
import static org.apache.hadoop.fs.s3a.Constants.*;
/**
* Look at the performance of S3a operations.
@ -71,6 +72,8 @@ public class TestS3AInputStreamPerformance extends S3AScaleTestBase {
@Before
public void openFS() throws IOException {
Configuration conf = getConf();
conf.setInt(SOCKET_SEND_BUFFER, 16 * 1024);
conf.setInt(SOCKET_RECV_BUFFER, 16 * 1024);
String testFile = conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE);
if (testFile.isEmpty()) {
assumptionMessage = "Empty test property: " + KEY_CSVTEST_FILE;