From c33d86860606f972f8b743b02f629b14f83d14f2 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Tue, 2 Jul 2024 11:34:45 +0100 Subject: [PATCH] HADOOP-19210. S3A: Speed up some slow unit tests (#6907) Speed up slow tests * TestS3AAWSCredentialsProvider: decrease thread pool shutdown time * TestS3AInputStreamRetry: reduce retry limit and intervals Contributed by Steve Loughran --- .../org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java | 9 +++++++++ .../hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java | 8 +++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java index f43710cf25..e76b304604 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java @@ -80,6 +80,15 @@ public Configuration createConfiguration() { conf.setInt(ASYNC_DRAIN_THRESHOLD, Integer.MAX_VALUE); // set the region to avoid the getBucketLocation on FS init. conf.set(AWS_REGION, "eu-west-1"); + + // tight retry logic as all failures are simulated + final String interval = "1ms"; + final int limit = 3; + conf.set(RETRY_THROTTLE_INTERVAL, interval); + conf.setInt(RETRY_THROTTLE_LIMIT, limit); + conf.set(RETRY_INTERVAL, interval); + conf.setInt(RETRY_LIMIT, limit); + return conf; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java index 0ffd7e75b1..d51bc954a6 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java @@ -86,6 +86,8 @@ public class TestS3AAWSCredentialsProvider extends AbstractS3ATestBase { private static final Logger LOG = LoggerFactory.getLogger(TestS3AAWSCredentialsProvider.class); + public static final int TERMINATION_TIMEOUT = 3; + @Test public void testProviderWrongClass() throws Exception { expectProviderInstantiationFailure(this.getClass(), @@ -579,7 +581,7 @@ protected AwsCredentials createCredentials(Configuration config) throws IOExcept } } - private static final int CONCURRENT_THREADS = 10; + private static final int CONCURRENT_THREADS = 4; @Test public void testConcurrentAuthentication() throws Throwable { @@ -619,7 +621,7 @@ public void testConcurrentAuthentication() throws Throwable { "expectedSecret", credentials.secretAccessKey()); } } finally { - pool.awaitTermination(10, TimeUnit.SECONDS); + pool.awaitTermination(TERMINATION_TIMEOUT, TimeUnit.SECONDS); pool.shutdown(); } @@ -685,7 +687,7 @@ public void testConcurrentAuthenticationError() throws Throwable { ); } } finally { - pool.awaitTermination(10, TimeUnit.SECONDS); + pool.awaitTermination(TERMINATION_TIMEOUT, TimeUnit.SECONDS); pool.shutdown(); }