diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index 242cd2fd2b..dfcb1b0fbe 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -34,6 +34,7 @@
UTF-8
true
+ ${project.build.directory}/test
@@ -59,7 +60,109 @@
false
+
+ parallel-tests
+
+
+
+ maven-antrun-plugin
+
+
+ create-parallel-tests-dirs
+ test-compile
+
+
+
+
+
+
+ run
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-surefire-plugin
+
+
+ default-test
+ test
+
+ test
+
+
+ ${testsThreadCount}
+ false
+ ${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true
+
+ ${test.build.data}/${surefire.forkNumber}
+ ${test.build.dir}/${surefire.forkNumber}
+ ${hadoop.tmp.dir}/${surefire.forkNumber}
+
+
+
+
+ fork-${surefire.forkNumber}
+
+
+
+
+
+
+
+
+
+
+
+
+
+ **/TestJets3tNativeS3FileSystemContract.java
+ **/TestS3ABlockingThreadPool.java
+ **/TestS3AFastOutputStream.java
+ **/TestS3AFileSystemContract.java
+ **/TestS3AMiniYarnCluster.java
+ **/Test*Root*.java
+
+
+
+
+ sequential-tests
+ test
+
+ test
+
+
+
+
+
+ **/TestJets3tNativeS3FileSystemContract.java
+ **/TestS3ABlockingThreadPool.java
+ **/TestS3AFastOutputStream.java
+ **/TestS3AFileSystemContract.java
+ **/TestS3AMiniYarnCluster.java
+ **/Test*Root*.java
+
+
+
+
+
+
+
+
@@ -116,30 +219,17 @@
hadoop-common
compile
-
org.apache.hadoop
hadoop-common
test
test-jar
-
-
-
- com.fasterxml.jackson.core
- jackson-databind
-
-
- com.fasterxml.jackson.core
- jackson-annotations
-
-
com.amazonaws
aws-java-sdk-s3
compile
-
junit
junit
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index 2259200918..95e3274dd7 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -730,3 +730,25 @@ or in batch runs.
Smaller values should result in faster test runs, especially when the object
store is a long way away.
+
+### Running the Tests
+
+After completing the configuration, execute the test run through Maven.
+
+ mvn clean test
+
+It's also possible to execute multiple test suites in parallel by enabling the
+`parallel-tests` Maven profile. The tests spend most of their time blocked on
+network I/O with the S3 service, so running in parallel tends to complete full
+test runs faster.
+
+ mvn -Pparallel-tests clean test
+
+Some tests must run with exclusive access to the S3 bucket, so even with the
+`parallel-tests` profile enabled, several test suites will run in serial in a
+separate Maven execution step after the parallel tests.
+
+By default, the `parallel-tests` profile runs 4 test suites concurrently. This
+can be tuned by passing the `testsThreadCount` argument.
+
+ mvn -Pparallel-tests -DtestsThreadCount=8 clean test
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/S3Contract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/S3Contract.java
index b388ce77af..483b78c9cb 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/S3Contract.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3/S3Contract.java
@@ -19,8 +19,12 @@
package org.apache.hadoop.fs.contract.s3;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.AbstractBondedFSContract;
+/**
+ * The contract of S3: only enabled if the test bucket is provided.
+ */
public class S3Contract extends AbstractBondedFSContract {
public static final String CONTRACT_XML = "contract/s3.xml";
@@ -37,4 +41,10 @@ public String getScheme() {
return "s3";
}
+ @Override
+ public Path getTestPath() {
+ String testUniqueForkId = System.getProperty("test.unique.fork.id");
+ return testUniqueForkId == null ? super.getTestPath() :
+ new Path("/" + testUniqueForkId, "test");
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java
index cbdb3bdb2c..e9024b5e82 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java
@@ -19,10 +19,11 @@
package org.apache.hadoop.fs.contract.s3a;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.AbstractBondedFSContract;
/**
- * The contract of S3A: only enabled if the test bucket is provided
+ * The contract of S3A: only enabled if the test bucket is provided.
*/
public class S3AContract extends AbstractBondedFSContract {
@@ -40,4 +41,10 @@ public String getScheme() {
return "s3a";
}
+ @Override
+ public Path getTestPath() {
+ String testUniqueForkId = System.getProperty("test.unique.fork.id");
+ return testUniqueForkId == null ? super.getTestPath() :
+ new Path("/" + testUniqueForkId, "test");
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRootDir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRootDir.java
index 5e2352c7eb..1cf1eb850c 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRootDir.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractRootDir.java
@@ -18,18 +18,55 @@
package org.apache.hadoop.fs.contract.s3a;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
- * root dir operations against an S3 bucket
+ * root dir operations against an S3 bucket.
*/
public class TestS3AContractRootDir extends
AbstractContractRootDirectoryTest {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestS3AContractRootDir.class);
+
@Override
protected AbstractFSContract createContract(Configuration conf) {
return new S3AContract(conf);
}
+
+ @Override
+ public void testListEmptyRootDirectory() throws IOException {
+ for (int attempt = 1, maxAttempts = 10; attempt <= maxAttempts; ++attempt) {
+ try {
+ super.testListEmptyRootDirectory();
+ break;
+ } catch (AssertionError | FileNotFoundException e) {
+ if (attempt < maxAttempts) {
+ LOG.info("Attempt {} of {} for empty root directory test failed. "
+ + "This is likely caused by eventual consistency of S3 "
+ + "listings. Attempting retry.", attempt, maxAttempts);
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e2) {
+ Thread.currentThread().interrupt();
+ fail("Test interrupted.");
+ break;
+ }
+ } else {
+ LOG.error(
+ "Empty root directory test failed {} attempts. Failing test.",
+ maxAttempts);
+ throw e;
+ }
+ }
+ }
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/NativeS3Contract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/NativeS3Contract.java
index ace6444b60..5796d88275 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/NativeS3Contract.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3n/NativeS3Contract.java
@@ -19,10 +19,11 @@
package org.apache.hadoop.fs.contract.s3n;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.AbstractBondedFSContract;
/**
- * The contract of S3N: only enabled if the test bucket is provided
+ * The contract of S3N: only enabled if the test bucket is provided.
*/
public class NativeS3Contract extends AbstractBondedFSContract {
@@ -40,4 +41,10 @@ public String getScheme() {
return "s3n";
}
+ @Override
+ public Path getTestPath() {
+ String testUniqueForkId = System.getProperty("test.unique.fork.id");
+ return testUniqueForkId == null ? super.getTestPath() :
+ new Path("/" + testUniqueForkId, "test");
+ }
}
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java
index 42c552a3bb..20ef6eca40 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java
@@ -127,7 +127,9 @@ public void tearDown() throws Exception {
}
protected Path getTestPath() {
- return new Path("/tests3a");
+ String testUniqueForkId = System.getProperty("test.unique.fork.id");
+ return testUniqueForkId == null ? new Path("/tests3a") :
+ new Path("/" + testUniqueForkId, "tests3a");
}
protected long getOperationCount() {