From b0aff8a96221fbf40d1eafe51b4f530b73146a20 Mon Sep 17 00:00:00 2001 From: Aaron Fabbri Date: Tue, 10 Apr 2018 23:55:38 -0700 Subject: [PATCH] HADOOP-15346 S3ARetryPolicy for 400/BadArgument to be "fail". Contributed by Steve Loughran. --- .../org/apache/hadoop/fs/s3a/S3ARetryPolicy.java | 6 +++--- .../java/org/apache/hadoop/fs/s3a/TestInvoker.java | 14 ++++---------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java index d857330395..2b361fd455 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java @@ -175,9 +175,9 @@ protected Map, RetryPolicy> createExceptionMap() { // which isn't going to be recovered from policyMap.put(EOFException.class, retryIdempotentCalls); - // policy on a 400/bad request still ambiguous. Given it - // comes and goes on test runs: try again - policyMap.put(AWSBadRequestException.class, connectivityFailure); + // policy on a 400/bad request still ambiguous. + // Treated as an immediate failure + policyMap.put(AWSBadRequestException.class, fail); // Status 500 error code is also treated as a connectivity problem policyMap.put(AWSStatus500Exception.class, connectivityFailure); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java index d29e2df50e..5da665c46b 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java @@ -283,18 +283,12 @@ public void testRetryAWSConnectivity() throws Throwable { /** * Repeatedly retry until eventually a bad request succeeds. */ - @Test - public void testRetryBadRequestIdempotent() throws Throwable { - final AtomicInteger counter = new AtomicInteger(0); - final int attemptsBeforeSuccess = ACTIVE_RETRY_LIMIT; - invoker.retry("test", null, true, + @Test(expected = AWSBadRequestException.class) + public void testRetryBadRequestNotIdempotent() throws Throwable { + invoker.retry("test", null, false, () -> { - if (counter.incrementAndGet() < attemptsBeforeSuccess) { - throw BAD_REQUEST; - } + throw BAD_REQUEST; }); - assertEquals(attemptsBeforeSuccess, counter.get()); - assertEquals("retry count ", attemptsBeforeSuccess - 1, retryCount); } @Test