big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-13868. [s3a] New default for S3A multi-part configuration (#1125)

Browse Source
This commit is contained in:
Sean Mackrory 2019-07-19 09:49:59 -06:00 committed by GitHub
parent cd967c75a7
commit 7f1b76ca35
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 6 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
View File

@ -1353,7 +1353,7 @@
<property>
<name>fs.s3a.multipart.size</name>
<value>100M</value>
<value>64M</value>
<description>How big (in bytes) to split upload or copy operations up into.
A suffix from the set {K,M,G,T,P} may be used to scale the numeric value.
</description>
@ -1361,7 +1361,7 @@
<property>
<name>fs.s3a.multipart.threshold</name>
<value>2147483647</value>
<value>128M</value>
<description>How big (in bytes) to split upload or copy operations up into.
This also controls the partition size in renamed files, as rename() involves
copying the source file(s).

4
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
View File

@ -202,12 +202,12 @@ private Constants() {
// size of each of or multipart pieces in bytes
public static final String MULTIPART_SIZE = "fs.s3a.multipart.size";
public static final long DEFAULT_MULTIPART_SIZE = 104857600; // 100 MB
public static final long DEFAULT_MULTIPART_SIZE = 67108864; // 64M
// minimum size in bytes before we start a multipart uploads or copy
public static final String MIN_MULTIPART_THRESHOLD =
"fs.s3a.multipart.threshold";
public static final long DEFAULT_MIN_MULTIPART_THRESHOLD = Integer.MAX_VALUE;
public static final long DEFAULT_MIN_MULTIPART_THRESHOLD = 134217728; // 128M
//enable multiobject-delete calls?
public static final String ENABLE_MULTI_DELETE =

4
hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
View File

@ -860,7 +860,7 @@ options are covered in [Testing](./testing.md).
<property>
<name>fs.s3a.multipart.size</name>
<value>100M</value>
<value>64M</value>
<description>How big (in bytes) to split upload or copy operations up into.
A suffix from the set {K,M,G,T,P} may be used to scale the numeric value.
</description>
@ -868,7 +868,7 @@ options are covered in [Testing](./testing.md).
<property>
<name>fs.s3a.multipart.threshold</name>
<value>2147483647</value>
<value>128MB</value>
<description>How big (in bytes) to split upload or copy operations up into.
This also controls the partition size in renamed files, as rename() involves
copying the source file(s).