big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-19210. S3A: Speed up some slow unit tests (#6907)

Browse Source 
Speed up slow tests
* TestS3AAWSCredentialsProvider: decrease thread pool shutdown time
* TestS3AInputStreamRetry: reduce retry limit and intervals

Contributed by Steve Loughran
This commit is contained in:
Steve Loughran 2024-07-02 11:34:45 +01:00 committed by GitHub
parent 134dcf166f
commit c33d868606
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 14 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java
View File

@ -80,6 +80,15 @@ public Configuration createConfiguration() {
conf.setInt(ASYNC_DRAIN_THRESHOLD, Integer.MAX_VALUE); conf.setInt(ASYNC_DRAIN_THRESHOLD, Integer.MAX_VALUE);
// set the region to avoid the getBucketLocation on FS init. // set the region to avoid the getBucketLocation on FS init.
conf.set(AWS_REGION, "eu-west-1"); conf.set(AWS_REGION, "eu-west-1");
// tight retry logic as all failures are simulated
final String interval = "1ms";
final int limit = 3;
conf.set(RETRY_THROTTLE_INTERVAL, interval);
conf.setInt(RETRY_THROTTLE_LIMIT, limit);
conf.set(RETRY_INTERVAL, interval);
conf.setInt(RETRY_LIMIT, limit);
return conf; return conf;
} }

8
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
View File

@ -86,6 +86,8 @@ public class TestS3AAWSCredentialsProvider extends AbstractS3ATestBase {
private static final Logger LOG = LoggerFactory.getLogger(TestS3AAWSCredentialsProvider.class); private static final Logger LOG = LoggerFactory.getLogger(TestS3AAWSCredentialsProvider.class);
public static final int TERMINATION_TIMEOUT = 3;
@Test @Test
public void testProviderWrongClass() throws Exception { public void testProviderWrongClass() throws Exception {
expectProviderInstantiationFailure(this.getClass(), expectProviderInstantiationFailure(this.getClass(),
@ -579,7 +581,7 @@ protected AwsCredentials createCredentials(Configuration config) throws IOExcept
} }
} }
private static final int CONCURRENT_THREADS = 10; private static final int CONCURRENT_THREADS = 4;
@Test @Test
public void testConcurrentAuthentication() throws Throwable { public void testConcurrentAuthentication() throws Throwable {
@ -619,7 +621,7 @@ public void testConcurrentAuthentication() throws Throwable {
"expectedSecret", credentials.secretAccessKey()); "expectedSecret", credentials.secretAccessKey());
} }
} finally { } finally {
pool.awaitTermination(10, TimeUnit.SECONDS); pool.awaitTermination(TERMINATION_TIMEOUT, TimeUnit.SECONDS);
pool.shutdown(); pool.shutdown();
} }
@ -685,7 +687,7 @@ public void testConcurrentAuthenticationError() throws Throwable {
); );
} }
} finally { } finally {
pool.awaitTermination(10, TimeUnit.SECONDS); pool.awaitTermination(TERMINATION_TIMEOUT, TimeUnit.SECONDS);
pool.shutdown(); pool.shutdown();
} }