diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index 242cd2fd2b..dfcb1b0fbe 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -34,6 +34,7 @@ UTF-8 true + ${project.build.directory}/test @@ -59,7 +60,109 @@ false + + parallel-tests + + + + maven-antrun-plugin + + + create-parallel-tests-dirs + test-compile + + + + + + + run + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + default-test + test + + test + + + ${testsThreadCount} + false + ${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true + + ${test.build.data}/${surefire.forkNumber} + ${test.build.dir}/${surefire.forkNumber} + ${hadoop.tmp.dir}/${surefire.forkNumber} + + + + + fork-${surefire.forkNumber} + + + + + + + + + + + + + + **/TestJets3tNativeS3FileSystemContract.java + **/TestS3ABlockingThreadPool.java + **/TestS3AFastOutputStream.java + **/TestS3AFileSystemContract.java + **/TestS3AMiniYarnCluster.java + **/Test*Root*.java + + + + + sequential-tests + test + + test + + + + + + **/TestJets3tNativeS3FileSystemContract.java + **/TestS3ABlockingThreadPool.java + **/TestS3AFastOutputStream.java + **/TestS3AFileSystemContract.java + **/TestS3AMiniYarnCluster.java + **/Test*Root*.java + + + + + + + + @@ -116,30 +219,17 @@ hadoop-common compile - org.apache.hadoop hadoop-common test test-jar - - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.core - jackson-annotations - - com.amazonaws aws-java-sdk-s3 compile - junit junit diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 2259200918..95e3274dd7 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -730,3 +730,25 @@ or in batch runs. Smaller values should result in faster test runs, especially when the object store is a long way away. + +### Running the Tests + +After completing the configuration, execute the test run through Maven. + + mvn clean test + +It's also possible to execute multiple test suites in parallel by enabling the +`parallel-tests` Maven profile. The tests spend most of their time blocked on +network I/O with the S3 service, so running in parallel tends to complete full +test runs faster. + + mvn -Pparallel-tests clean test + +Some tests must run with exclusive access to the S3 bucket, so even with the +`parallel-tests` profile enabled, several test suites will run in serial in a +separate Maven execution step after the parallel tests. + +By default, the `parallel-tests` profile runs 4 test suites concurrently. This +can be tuned by passing the `testsThreadCount` argument. + + mvn -Pparallel-tests -DtestsThreadCount=8 clean test diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/S3Contract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/S3Contract.java index b388ce77af..483b78c9cb 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/S3Contract.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/S3Contract.java @@ -19,8 +19,12 @@ package org.apache.hadoop.fs.contract.s3; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.AbstractBondedFSContract; +/** + * The contract of S3: only enabled if the test bucket is provided. + */ public class S3Contract extends AbstractBondedFSContract { public static final String CONTRACT_XML = "contract/s3.xml"; @@ -37,4 +41,10 @@ public String getScheme() { return "s3"; } + @Override + public Path getTestPath() { + String testUniqueForkId = System.getProperty("test.unique.fork.id"); + return testUniqueForkId == null ? super.getTestPath() : + new Path("/" + testUniqueForkId, "test"); + } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java index cbdb3bdb2c..e9024b5e82 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java @@ -19,10 +19,11 @@ package org.apache.hadoop.fs.contract.s3a; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.AbstractBondedFSContract; /** - * The contract of S3A: only enabled if the test bucket is provided + * The contract of S3A: only enabled if the test bucket is provided. */ public class S3AContract extends AbstractBondedFSContract { @@ -40,4 +41,10 @@ public String getScheme() { return "s3a"; } + @Override + public Path getTestPath() { + String testUniqueForkId = System.getProperty("test.unique.fork.id"); + return testUniqueForkId == null ? super.getTestPath() : + new Path("/" + testUniqueForkId, "test"); + } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRootDir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRootDir.java index 5e2352c7eb..1cf1eb850c 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRootDir.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRootDir.java @@ -18,18 +18,55 @@ package org.apache.hadoop.fs.contract.s3a; +import java.io.FileNotFoundException; +import java.io.IOException; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest; import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** - * root dir operations against an S3 bucket + * root dir operations against an S3 bucket. */ public class TestS3AContractRootDir extends AbstractContractRootDirectoryTest { + private static final Logger LOG = + LoggerFactory.getLogger(TestS3AContractRootDir.class); + @Override protected AbstractFSContract createContract(Configuration conf) { return new S3AContract(conf); } + + @Override + public void testListEmptyRootDirectory() throws IOException { + for (int attempt = 1, maxAttempts = 10; attempt <= maxAttempts; ++attempt) { + try { + super.testListEmptyRootDirectory(); + break; + } catch (AssertionError | FileNotFoundException e) { + if (attempt < maxAttempts) { + LOG.info("Attempt {} of {} for empty root directory test failed. " + + "This is likely caused by eventual consistency of S3 " + + "listings. Attempting retry.", attempt, maxAttempts); + try { + Thread.sleep(1000); + } catch (InterruptedException e2) { + Thread.currentThread().interrupt(); + fail("Test interrupted."); + break; + } + } else { + LOG.error( + "Empty root directory test failed {} attempts. Failing test.", + maxAttempts); + throw e; + } + } + } + } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/NativeS3Contract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/NativeS3Contract.java index ace6444b60..5796d88275 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/NativeS3Contract.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/NativeS3Contract.java @@ -19,10 +19,11 @@ package org.apache.hadoop.fs.contract.s3n; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.AbstractBondedFSContract; /** - * The contract of S3N: only enabled if the test bucket is provided + * The contract of S3N: only enabled if the test bucket is provided. */ public class NativeS3Contract extends AbstractBondedFSContract { @@ -40,4 +41,10 @@ public String getScheme() { return "s3n"; } + @Override + public Path getTestPath() { + String testUniqueForkId = System.getProperty("test.unique.fork.id"); + return testUniqueForkId == null ? super.getTestPath() : + new Path("/" + testUniqueForkId, "test"); + } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java index 42c552a3bb..20ef6eca40 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java @@ -127,7 +127,9 @@ public void tearDown() throws Exception { } protected Path getTestPath() { - return new Path("/tests3a"); + String testUniqueForkId = System.getProperty("test.unique.fork.id"); + return testUniqueForkId == null ? new Path("/tests3a") : + new Path("/" + testUniqueForkId, "tests3a"); } protected long getOperationCount() {