From 45d1b0fdcc04a86be91a9b72073cdc30bec04d3b Mon Sep 17 00:00:00 2001 From: Chris Douglas Date: Mon, 12 Mar 2018 19:47:42 -0700 Subject: [PATCH] HADOOP-14696. parallel tests don't work for Windows. Contributed by Allen Wittenauer --- hadoop-common-project/hadoop-common/pom.xml | 26 +---- .../apache/hadoop/test/GenericTestUtils.java | 68 ++++++++---- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 26 +---- .../plugin/paralleltests/CreateDirsMojo.java | 100 ++++++++++++++++++ hadoop-tools/hadoop-aws/pom.xml | 26 +---- 5 files changed, 161 insertions(+), 85 deletions(-) create mode 100644 hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/paralleltests/CreateDirsMojo.java diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 078a94335f..49d3575d3c 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -979,30 +979,13 @@ - maven-antrun-plugin + org.apache.hadoop + hadoop-maven-plugins - create-parallel-tests-dirs - test-compile - - - - - + parallel-tests-createdir - run + parallel-tests-createdir @@ -1015,6 +998,7 @@ false ${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true + ${testsThreadCount} ${test.build.data}/${surefire.forkNumber} ${test.build.dir}/${surefire.forkNumber} ${hadoop.tmp.dir}/${surefire.forkNumber} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java index cdde48ce57..61b0271d98 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java @@ -286,7 +286,7 @@ public static String getRandomizedTempPath() { public static void assertExists(File f) { Assert.assertTrue("File " + f + " should exist", f.exists()); } - + /** * List all of the files in 'dir' that match the regex 'pattern'. * Then check that this list is identical to 'expectedMatches'. @@ -294,7 +294,7 @@ public static void assertExists(File f) { */ public static void assertGlobEquals(File dir, String pattern, String ... expectedMatches) throws IOException { - + Set found = Sets.newTreeSet(); for (File f : FileUtil.listFiles(dir)) { if (f.getName().matches(pattern)) { @@ -349,7 +349,7 @@ public static void assertExceptionContains(String expectedText, StringUtils.stringifyException(t)), t); } - } + } /** * Wait for the specified test to return true. The test will be performed @@ -499,18 +499,18 @@ public void clearOutput() { */ public static class DelayAnswer implements Answer { private final Log LOG; - + private final CountDownLatch fireLatch = new CountDownLatch(1); private final CountDownLatch waitLatch = new CountDownLatch(1); private final CountDownLatch resultLatch = new CountDownLatch(1); - + private final AtomicInteger fireCounter = new AtomicInteger(0); private final AtomicInteger resultCounter = new AtomicInteger(0); - + // Result fields set after proceed() is called. private volatile Throwable thrown; private volatile Object returnValue; - + public DelayAnswer(Log log) { this.LOG = log; } @@ -521,7 +521,7 @@ public DelayAnswer(Log log) { public void waitForCall() throws InterruptedException { fireLatch.await(); } - + /** * Tell the method to proceed. * This should only be called after waitForCall() @@ -529,7 +529,7 @@ public void waitForCall() throws InterruptedException { public void proceed() { waitLatch.countDown(); } - + @Override public Object answer(InvocationOnMock invocation) throws Throwable { LOG.info("DelayAnswer firing fireLatch"); @@ -558,7 +558,7 @@ protected Object passThrough(InvocationOnMock invocation) throws Throwable { resultLatch.countDown(); } } - + /** * After calling proceed(), this will wait until the call has * completed and a result has been returned to the caller. @@ -566,7 +566,7 @@ protected Object passThrough(InvocationOnMock invocation) throws Throwable { public void waitForResult() throws InterruptedException { resultLatch.await(); } - + /** * After the call has gone through, return any exception that * was thrown, or null if no exception was thrown. @@ -574,7 +574,7 @@ public void waitForResult() throws InterruptedException { public Throwable getThrown() { return thrown; } - + /** * After the call has gone through, return the call's return value, * or null in case it was void or an exception was thrown. @@ -582,20 +582,20 @@ public Throwable getThrown() { public Object getReturnValue() { return returnValue; } - + public int getFireCount() { return fireCounter.get(); } - + public int getResultCount() { return resultCounter.get(); } } - + /** * An Answer implementation that simply forwards all calls through * to a delegate. - * + * * This is useful as the default Answer for a mock object, to create * something like a spy on an RPC proxy. For example: * @@ -606,14 +606,14 @@ public int getResultCount() { * ... * */ - public static class DelegateAnswer implements Answer { + public static class DelegateAnswer implements Answer { private final Object delegate; private final Log log; - + public DelegateAnswer(Object delegate) { this(null, delegate); } - + public DelegateAnswer(Log log, Object delegate) { this.log = log; this.delegate = delegate; @@ -653,7 +653,7 @@ public SleepAnswer(int minSleepTime, int maxSleepTime) { this.minSleepTime = minSleepTime; this.maxSleepTime = maxSleepTime; } - + @Override public Object answer(InvocationOnMock invocation) throws Throwable { boolean interrupted = false; @@ -683,11 +683,11 @@ public static void assertMatches(String output, String pattern) { " but got:\n" + output, Pattern.compile(pattern).matcher(output).find()); } - + public static void assertValueNear(long expected, long actual, long allowedError) { assertValueWithinRange(expected - allowedError, expected + allowedError, actual); } - + public static void assertValueWithinRange(long expectedMin, long expectedMax, long actual) { Assert.assertTrue("Expected " + actual + " to be in range (" + expectedMin + "," @@ -842,4 +842,28 @@ public static void failif(boolean condition, failf(format, args); } } + + /** + * Retreive the max number of parallel test threads when running under maven. + * @return int number of threads + */ + public static int getTestsThreadCount() { + String propString = System.getProperty("testsThreadCount", "1"); + int threadCount = 1; + if (propString != null) { + String trimProp = propString.trim(); + if (trimProp.endsWith("C")) { + double multiplier = Double.parseDouble( + trimProp.substring(0, trimProp.length()-1)); + double calculated = multiplier * ((double) Runtime + .getRuntime() + .availableProcessors()); + threadCount = calculated > 0d ? Math.max((int) calculated, 1) : 0; + } else { + threadCount = Integer.parseInt(trimProp); + } + } + return threadCount; + } + } diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index d6afed1fba..909e5890ed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -515,30 +515,13 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> - maven-antrun-plugin + org.apache.hadoop + hadoop-maven-plugins - create-parallel-tests-dirs - test-compile - - - - - + parallel-tests-createdir - run + parallel-tests-createdir @@ -551,6 +534,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> false ${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true + ${testsThreadCount} ${test.build.data}/${surefire.forkNumber} ${test.build.dir}/${surefire.forkNumber} ${hadoop.tmp.dir}/${surefire.forkNumber} diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/paralleltests/CreateDirsMojo.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/paralleltests/CreateDirsMojo.java new file mode 100644 index 0000000000..4c9b9d7317 --- /dev/null +++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/paralleltests/CreateDirsMojo.java @@ -0,0 +1,100 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.maven.plugin.paralleltests; + +import java.io.File; + +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugins.annotations.LifecyclePhase; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; + + +/** + * Goal which creates the parallel-test directories. + */ +@Mojo(name="parallel-tests-createdir", + defaultPhase = LifecyclePhase.GENERATE_TEST_RESOURCES) +public class CreateDirsMojo extends AbstractMojo { + + /** + * Location of the test.build.dir. + */ + @Parameter(defaultValue="${project.build.directory}/test-dir") + private File testBuildDir; + + /** + * Location of the test.build.data. + */ + @Parameter(defaultValue="${project.build.directory}/test-dir") + private File testBuildData; + + /** + * Location of the test.build.data. + */ + @Parameter(defaultValue="${project.build.directory}/tmp") + private File hadoopTmpDir; + + /** + * Thread count. + */ + @Parameter(defaultValue="${testsThreadCount}") + private String testsThreadCount; + + public void execute() throws MojoExecutionException { + int numDirs=getTestsThreadCount(); + + mkParallelDirs(testBuildDir, numDirs); + mkParallelDirs(testBuildData, numDirs); + mkParallelDirs(hadoopTmpDir, numDirs); + } + + /** + * Get the real number of parallel threads. + * @return int number of threads + */ + + public int getTestsThreadCount() { + int threadCount = 1; + if (testsThreadCount != null) { + String trimProp = testsThreadCount.trim(); + if (trimProp.endsWith("C")) { + double multiplier = Double.parseDouble( + trimProp.substring(0, trimProp.length()-1)); + double calculated = multiplier * ((double) Runtime + .getRuntime() + .availableProcessors()); + threadCount = calculated > 0d ? Math.max((int) calculated, 1) : 0; + } else { + threadCount = Integer.parseInt(testsThreadCount); + } + } + return threadCount; + } + + private void mkParallelDirs(File testDir, int numDirs) + throws MojoExecutionException { + for (int i=1; i<=numDirs; i++) { + File newDir = new File(testDir, String.valueOf(i)); + if (!newDir.exists()) { + getLog().info("Creating " + newDir.toString()); + if (!newDir.mkdirs()) { + throw new MojoExecutionException("Unable to create " + + newDir.toString()); + } + } + } + } +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index c704a629eb..24ed11dee4 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -85,30 +85,13 @@ - maven-antrun-plugin + org.apache.hadoop + hadoop-maven-plugins - create-parallel-tests-dirs - test-compile - - - - - + parallel-tests-createdir - run + parallel-tests-createdir @@ -121,6 +104,7 @@ false ${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true + ${testsThreadCount} ${test.build.data}/${surefire.forkNumber} ${test.build.dir}/${surefire.forkNumber} ${hadoop.tmp.dir}/${surefire.forkNumber}