big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-18380. fs.s3a.prefetch.block.size to be read through longBytesOption (#4762)

Browse Source 
Contributed by Viraj Jasani.
This commit is contained in:
Viraj Jasani 2022-08-23 02:49:04 -07:00 committed by GitHub
parent eda4bb5dcd
commit c249db80c2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
View File

@ -513,8 +513,13 @@ public void initialize(URI name, Configuration originalConf)
enableMultiObjectsDelete = conf.getBoolean(ENABLE_MULTI_DELETE, true);
this.prefetchEnabled = conf.getBoolean(PREFETCH_ENABLED_KEY, PREFETCH_ENABLED_DEFAULT);
this.prefetchBlockSize = intOption(
conf, PREFETCH_BLOCK_SIZE_KEY, PREFETCH_BLOCK_DEFAULT_SIZE, PREFETCH_BLOCK_DEFAULT_SIZE);
long prefetchBlockSizeLong =
longBytesOption(conf, PREFETCH_BLOCK_SIZE_KEY, PREFETCH_BLOCK_DEFAULT_SIZE,
PREFETCH_BLOCK_DEFAULT_SIZE);
if (prefetchBlockSizeLong > (long) Integer.MAX_VALUE) {
throw new IOException("S3A prefatch block size exceeds int limit");
}
this.prefetchBlockSize = (int) prefetchBlockSizeLong;
this.prefetchBlockCount =
intOption(conf, PREFETCH_BLOCK_COUNT_KEY, PREFETCH_BLOCK_DEFAULT_COUNT, 1);