big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-13868. [s3a] New default for S3A multi-part configuration (#1125)

Browse Source
This commit is contained in:
Sean Mackrory 2019-07-19 09:49:59 -06:00 committed by GitHub
parent cd967c75a7
commit 7f1b76ca35
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 6 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
View File

@ -1353,7 +1353,7 @@
<property> <property>
<name>fs.s3a.multipart.size</name> <name>fs.s3a.multipart.size</name>
<value>100M</value> <value>64M</value>
<description>How big (in bytes) to split upload or copy operations up into. <description>How big (in bytes) to split upload or copy operations up into.
A suffix from the set {K,M,G,T,P} may be used to scale the numeric value. A suffix from the set {K,M,G,T,P} may be used to scale the numeric value.
</description> </description>
@ -1361,7 +1361,7 @@
<property> <property>
<name>fs.s3a.multipart.threshold</name> <name>fs.s3a.multipart.threshold</name>
<value>2147483647</value> <value>128M</value>
<description>How big (in bytes) to split upload or copy operations up into. <description>How big (in bytes) to split upload or copy operations up into.
This also controls the partition size in renamed files, as rename() involves This also controls the partition size in renamed files, as rename() involves
copying the source file(s). copying the source file(s).

4
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
View File

@ -202,12 +202,12 @@ private Constants() {
// size of each of or multipart pieces in bytes // size of each of or multipart pieces in bytes
public static final String MULTIPART_SIZE = "fs.s3a.multipart.size"; public static final String MULTIPART_SIZE = "fs.s3a.multipart.size";
public static final long DEFAULT_MULTIPART_SIZE = 104857600; // 100 MB public static final long DEFAULT_MULTIPART_SIZE = 67108864; // 64M
// minimum size in bytes before we start a multipart uploads or copy // minimum size in bytes before we start a multipart uploads or copy
public static final String MIN_MULTIPART_THRESHOLD = public static final String MIN_MULTIPART_THRESHOLD =
"fs.s3a.multipart.threshold"; "fs.s3a.multipart.threshold";
public static final long DEFAULT_MIN_MULTIPART_THRESHOLD = Integer.MAX_VALUE; public static final long DEFAULT_MIN_MULTIPART_THRESHOLD = 134217728; // 128M
//enable multiobject-delete calls? //enable multiobject-delete calls?
public static final String ENABLE_MULTI_DELETE = public static final String ENABLE_MULTI_DELETE =

4
hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
View File

@ -860,7 +860,7 @@ options are covered in [Testing](./testing.md).
<property> <property>
<name>fs.s3a.multipart.size</name> <name>fs.s3a.multipart.size</name>
<value>100M</value> <value>64M</value>
<description>How big (in bytes) to split upload or copy operations up into. <description>How big (in bytes) to split upload or copy operations up into.
A suffix from the set {K,M,G,T,P} may be used to scale the numeric value. A suffix from the set {K,M,G,T,P} may be used to scale the numeric value.
</description> </description>
@ -868,7 +868,7 @@ options are covered in [Testing](./testing.md).
<property> <property>
<name>fs.s3a.multipart.threshold</name> <name>fs.s3a.multipart.threshold</name>
<value>2147483647</value> <value>128MB</value>
<description>How big (in bytes) to split upload or copy operations up into. <description>How big (in bytes) to split upload or copy operations up into.
This also controls the partition size in renamed files, as rename() involves This also controls the partition size in renamed files, as rename() involves
copying the source file(s). copying the source file(s).