big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-17458. S3A to treat "SdkClientException: Data read has a different length than the expected" as EOFException (#3040)

Browse Source 
Some network exceptions can raise SdkClientException with message
`Data read has a different length than the expected`.

These should be recoverable.

Contributed by Bogdan Stolojan

Change-Id: Ia22fd77d90971e9e02b4f947398a4749eebe5909
This commit is contained in:
Petre Bogdan Stolojan 2021-07-23 14:44:29 +01:00 committed by Steve Loughran
parent 5ea1337167
commit e89d30b6b7
No known key found for this signature in database
GPG Key ID: D22CF846DBB162A0
2 changed files with 46 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
View File

@ -132,9 +132,12 @@ public final class S3AUtils {
S3AEncryptionMethods.SSE_S3.getMethod()
+ " is enabled but an encryption key was set in "
+ SERVER_SIDE_ENCRYPTION_KEY;
private static final String EOF_MESSAGE_IN_XML_PARSER
public static final String EOF_MESSAGE_IN_XML_PARSER
= "Failed to sanitize XML document destined for handler class";
public static final String EOF_READ_DIFFERENT_LENGTH
= "Data read has a different length than the expected";
private static final String BUCKET_PATTERN = FS_S3A_BUCKET_PREFIX + "%s.%s";
/**
@ -194,7 +197,7 @@ public static IOException translateException(@Nullable String operation,
// interrupted IO, or a socket exception underneath that class
return translateInterruptedException(exception, innerCause, message);
}
if (signifiesConnectionBroken(exception)) {
if (isMessageTranslatableToEOF(exception)) {
// call considered an sign of connectivity failure
return (EOFException)new EOFException(message).initCause(exception);
}
@ -415,13 +418,14 @@ public static boolean isThrottleException(Exception ex) {
/**
* Cue that an AWS exception is likely to be an EOF Exception based
* on the message coming back from an XML/JSON parser. This is likely
* to be brittle, so only a hint.
* on the message coming back from the client. This is likely to be
* brittle, so only a hint.
* @param ex exception
* @return true if this is believed to be a sign the connection was broken.
*/
public static boolean signifiesConnectionBroken(SdkBaseException ex) {
return ex.toString().contains(EOF_MESSAGE_IN_XML_PARSER);
public static boolean isMessageTranslatableToEOF(SdkBaseException ex) {
return ex.toString().contains(EOF_MESSAGE_IN_XML_PARSER) ||
ex.toString().contains(EOF_READ_DIFFERENT_LENGTH);
}
/**

36
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.fs.s3a;
import java.io.EOFException;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.net.SocketTimeoutException;
@ -28,6 +29,7 @@
import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.SdkBaseException;
import com.amazonaws.SdkClientException;
import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughputExceededException;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import org.junit.Assert;
@ -163,6 +165,40 @@ public void test500isStatus500Exception() throws Exception {
ex);
}
@Test
public void testExceptionsWithTranslatableMessage() throws Exception {
SdkBaseException xmlParsing = new SdkBaseException(EOF_MESSAGE_IN_XML_PARSER);
SdkBaseException differentLength = new SdkBaseException(EOF_READ_DIFFERENT_LENGTH);
verifyTranslated(EOFException.class, xmlParsing);
verifyTranslated(EOFException.class, differentLength);
}
@Test
public void testSdkDifferentLengthExceptionIsTranslatable() throws Throwable {
final AtomicInteger counter = new AtomicInteger(0);
invoker.retry("test", null, false, () -> {
if (counter.incrementAndGet() < ACTIVE_RETRY_LIMIT) {
throw new SdkClientException(EOF_READ_DIFFERENT_LENGTH);
}
});
assertEquals(ACTIVE_RETRY_LIMIT, counter.get());
}
@Test
public void testSdkXmlParsingExceptionIsTranslatable() throws Throwable {
final AtomicInteger counter = new AtomicInteger(0);
invoker.retry("test", null, false, () -> {
if (counter.incrementAndGet() < ACTIVE_RETRY_LIMIT) {
throw new SdkClientException(EOF_MESSAGE_IN_XML_PARSER);
}
});
assertEquals(ACTIVE_RETRY_LIMIT, counter.get());
}
@Test(expected = org.apache.hadoop.net.ConnectTimeoutException.class)
public void testExtractConnectTimeoutException() throws Throwable {
throw extractException("", "",