big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-15346 S3ARetryPolicy for 400/BadArgument to be "fail". Contributed by Steve Loughran.

Browse Source
This commit is contained in:
Aaron Fabbri 2018-04-10 23:55:38 -07:00
parent 7c9cdad6d0
commit b0aff8a962
No known key found for this signature in database
GPG Key ID: B2EEFA9E78118A29
2 changed files with 7 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java
View File

@ -175,9 +175,9 @@ protected Map<Class<? extends Exception>, RetryPolicy> createExceptionMap() {
// which isn't going to be recovered from
policyMap.put(EOFException.class, retryIdempotentCalls);
// policy on a 400/bad request still ambiguous. Given it
// comes and goes on test runs: try again
policyMap.put(AWSBadRequestException.class, connectivityFailure);
// policy on a 400/bad request still ambiguous.
// Treated as an immediate failure
policyMap.put(AWSBadRequestException.class, fail);
// Status 500 error code is also treated as a connectivity problem
policyMap.put(AWSStatus500Exception.class, connectivityFailure);

14
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java
View File

@ -283,18 +283,12 @@ public void testRetryAWSConnectivity() throws Throwable {
/**
* Repeatedly retry until eventually a bad request succeeds.
*/
@Test
public void testRetryBadRequestIdempotent() throws Throwable {
final AtomicInteger counter = new AtomicInteger(0);
final int attemptsBeforeSuccess = ACTIVE_RETRY_LIMIT;
invoker.retry("test", null, true,
@Test(expected = AWSBadRequestException.class)
public void testRetryBadRequestNotIdempotent() throws Throwable {
invoker.retry("test", null, false,
() -> {
if (counter.incrementAndGet() < attemptsBeforeSuccess) {
throw BAD_REQUEST;
}
throw BAD_REQUEST;
});
assertEquals(attemptsBeforeSuccess, counter.get());
assertEquals("retry count ", attemptsBeforeSuccess - 1, retryCount);
}
@Test