From 42eb9ff68e3786dce44a89e78d9a5dc3603ec2fc Mon Sep 17 00:00:00 2001 From: Gabor Bota Date: Tue, 5 Jan 2021 15:43:01 +0100 Subject: [PATCH] HADOOP-17454. [s3a] Disable bucket existence check - set fs.s3a.bucket.probe to 0 (#2593) Also fixes HADOOP-16995. ITestS3AConfiguration proxy tests failures when bucket probes == 0 The improvement should include the fix, ebcause the test would fail by default otherwise. Change-Id: I9a7e4b5e6d4391ebba096c15e84461c038a2ec59 --- .../src/main/java/org/apache/hadoop/fs/s3a/Constants.java | 2 +- .../hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md | 4 ++-- .../java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index fcaec50929..49a0f8105a 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -499,7 +499,7 @@ private Constants() { * will be validated using {@code S3AFileSystem.verifyBucketExistsV2()}. * Value: {@value} */ - public static final int S3A_BUCKET_PROBE_DEFAULT = 2; + public static final int S3A_BUCKET_PROBE_DEFAULT = 0; /** * How long a directory listing in the MS is considered as authoritative. diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 861da4d82e..2158ee5042 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -1039,9 +1039,9 @@ options are covered in [Testing](./testing.md). fs.s3a.bucket.probe - 2 + 0 - The value can be 0, 1 or 2 (default). + The value can be 0 (default), 1 or 2. When set to 0, bucket existence checks won't be done during initialization thus making it faster. Though it should be noted that when the bucket is not available in S3, diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java index 57cbbcbe85..eb68eed1be 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java @@ -159,6 +159,7 @@ private E expectFSCreateFailure( return intercept(clazz, () -> { fs = S3ATestUtils.createTestFileSystem(conf); + fs.listFiles(new Path("/"), false); return "expected failure creating FS " + text + " got " + fs; }); }