big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-13212 Provide an option to set the socket buffers in S3AFileSystem (Rajesh Balamohan)

Browse Source
This commit is contained in:
Steve Loughran 2016-07-20 13:42:19 +01:00
parent 9ccf935b2c
commit 37362c2f92
5 changed files with 40 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
View File

@ -907,6 +907,18 @@
<description>Socket connection timeout in milliseconds.</description> <description>Socket connection timeout in milliseconds.</description>
</property> </property>
<property>
<name>fs.s3a.socket.send.buffer</name>
<value>8192</value>
<description>Socket send buffer hint to amazon connector. Represented in bytes.</description>
</property>
<property>
<name>fs.s3a.socket.recv.buffer</name>
<value>8192</value>
<description>Socket receive buffer hint to amazon connector. Represented in bytes.</description>
</property>
<property> <property>
<name>fs.s3a.paging.maximum</name> <name>fs.s3a.paging.maximum</name>
<value>5000</value> <value>5000</value>

8
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
View File

@ -82,6 +82,14 @@ private Constants() {
public static final String SOCKET_TIMEOUT = "fs.s3a.connection.timeout"; public static final String SOCKET_TIMEOUT = "fs.s3a.connection.timeout";
public static final int DEFAULT_SOCKET_TIMEOUT = 200000; public static final int DEFAULT_SOCKET_TIMEOUT = 200000;
// socket send buffer to be used in Amazon client
public static final String SOCKET_SEND_BUFFER = "fs.s3a.socket.send.buffer";
public static final int DEFAULT_SOCKET_SEND_BUFFER = 8 * 1024;
// socket send buffer to be used in Amazon client
public static final String SOCKET_RECV_BUFFER = "fs.s3a.socket.recv.buffer";
public static final int DEFAULT_SOCKET_RECV_BUFFER = 8 * 1024;
// number of records to get while paging through a directory listing // number of records to get while paging through a directory listing
public static final String MAX_PAGING_KEYS = "fs.s3a.paging.maximum"; public static final String MAX_PAGING_KEYS = "fs.s3a.paging.maximum";
public static final int DEFAULT_MAX_PAGING_KEYS = 5000; public static final int DEFAULT_MAX_PAGING_KEYS = 5000;

5
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
View File

@ -163,6 +163,11 @@ public void initialize(URI name, Configuration conf) throws IOException {
DEFAULT_ESTABLISH_TIMEOUT, 0)); DEFAULT_ESTABLISH_TIMEOUT, 0));
awsConf.setSocketTimeout(intOption(conf, SOCKET_TIMEOUT, awsConf.setSocketTimeout(intOption(conf, SOCKET_TIMEOUT,
DEFAULT_SOCKET_TIMEOUT, 0)); DEFAULT_SOCKET_TIMEOUT, 0));
int sockSendBuffer = intOption(conf, SOCKET_SEND_BUFFER,
DEFAULT_SOCKET_SEND_BUFFER, 2048);
int sockRecvBuffer = intOption(conf, SOCKET_RECV_BUFFER,
DEFAULT_SOCKET_RECV_BUFFER, 2048);
awsConf.setSocketBufferSizeHints(sockSendBuffer, sockRecvBuffer);
String signerOverride = conf.getTrimmed(SIGNING_ALGORITHM, ""); String signerOverride = conf.getTrimmed(SIGNING_ALGORITHM, "");
if (!signerOverride.isEmpty()) { if (!signerOverride.isEmpty()) {
LOG.debug("Signer override = {}", signerOverride); LOG.debug("Signer override = {}", signerOverride);

12
hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
View File

@ -503,6 +503,18 @@ this capability.
which each use a thread from the threadpool.</description> which each use a thread from the threadpool.</description>
</property> </property>
<property>
<name>fs.s3a.socket.send.buffer</name>
<value>8192</value>
<description>Socket send buffer hint to amazon connector. Represented in bytes.</description>
</property>
<property>
<name>fs.s3a.socket.recv.buffer</name>
<value>8192</value>
<description>Socket receive buffer hint to amazon connector. Represented in bytes.</description>
</property>
<property> <property>
<name>fs.s3a.threads.keepalivetime</name> <name>fs.s3a.threads.keepalivetime</name>
<value>60</value> <value>60</value>

3
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/TestS3AInputStreamPerformance.java
View File

@ -44,6 +44,7 @@
import java.io.IOException; import java.io.IOException;
import static org.apache.hadoop.fs.contract.ContractTestUtils.*; import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
import static org.apache.hadoop.fs.s3a.Constants.*;
/** /**
* Look at the performance of S3a operations. * Look at the performance of S3a operations.
@ -71,6 +72,8 @@ public class TestS3AInputStreamPerformance extends S3AScaleTestBase {
@Before @Before
public void openFS() throws IOException { public void openFS() throws IOException {
Configuration conf = getConf(); Configuration conf = getConf();
conf.setInt(SOCKET_SEND_BUFFER, 16 * 1024);
conf.setInt(SOCKET_RECV_BUFFER, 16 * 1024);
String testFile = conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE); String testFile = conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE);
if (testFile.isEmpty()) { if (testFile.isEmpty()) {
assumptionMessage = "Empty test property: " + KEY_CSVTEST_FILE; assumptionMessage = "Empty test property: " + KEY_CSVTEST_FILE;