big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-14696. parallel tests don't work for Windows. Contributed by Allen Wittenauer

Browse Source
This commit is contained in:
Chris Douglas 2018-03-12 19:47:42 -07:00
parent 19292bc264
commit 45d1b0fdcc
5 changed files with 161 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
hadoop-common-project/hadoop-common/pom.xml
View File

@ -979,30 +979,13 @@
<build>
<plugins>
<plugin>
<artifactId>maven-antrun-plugin</artifactId>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-maven-plugins</artifactId>
<executions>
<execution>
<id>create-parallel-tests-dirs</id>
<phase>test-compile</phase>
<configuration>
<target>
<script language="javascript"><![CDATA[
var baseDirs = [
"${test.build.data}",
"${test.build.dir}",
"${hadoop.tmp.dir}" ];
for (var i in baseDirs) {
for (var j = 1; j <= ${testsThreadCount}; ++j) {
var mkdir = project.createTask("mkdir");
mkdir.setDir(new java.io.File(baseDirs[i], j));
mkdir.perform();
}
}
]]></script>
</target>
</configuration>
<id>parallel-tests-createdir</id>
<goals>
<goal>run</goal>
<goal>parallel-tests-createdir</goal>
</goals>
</execution>
</executions>
@ -1015,6 +998,7 @@
<reuseForks>false</reuseForks>
<argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
<systemPropertyVariables>
<testsThreadCount>${testsThreadCount}</testsThreadCount>
<test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
<test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
<hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>

68
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
View File

@ -286,7 +286,7 @@ public static String getRandomizedTempPath() {
public static void assertExists(File f) {
Assert.assertTrue("File " + f + " should exist", f.exists());
}
/**
* List all of the files in 'dir' that match the regex 'pattern'.
* Then check that this list is identical to 'expectedMatches'.
@ -294,7 +294,7 @@ public static void assertExists(File f) {
*/
public static void assertGlobEquals(File dir, String pattern,
String ... expectedMatches) throws IOException {
Set<String> found = Sets.newTreeSet();
for (File f : FileUtil.listFiles(dir)) {
if (f.getName().matches(pattern)) {
@ -349,7 +349,7 @@ public static void assertExceptionContains(String expectedText,
StringUtils.stringifyException(t)),
t);
}
}
}
/**
* Wait for the specified test to return true. The test will be performed
@ -499,18 +499,18 @@ public void clearOutput() {
*/
public static class DelayAnswer implements Answer<Object> {
private final Log LOG;
private final CountDownLatch fireLatch = new CountDownLatch(1);
private final CountDownLatch waitLatch = new CountDownLatch(1);
private final CountDownLatch resultLatch = new CountDownLatch(1);
private final AtomicInteger fireCounter = new AtomicInteger(0);
private final AtomicInteger resultCounter = new AtomicInteger(0);
// Result fields set after proceed() is called.
private volatile Throwable thrown;
private volatile Object returnValue;
public DelayAnswer(Log log) {
this.LOG = log;
}
@ -521,7 +521,7 @@ public DelayAnswer(Log log) {
public void waitForCall() throws InterruptedException {
fireLatch.await();
}
/**
* Tell the method to proceed.
* This should only be called after waitForCall()
@ -529,7 +529,7 @@ public void waitForCall() throws InterruptedException {
public void proceed() {
waitLatch.countDown();
}
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
LOG.info("DelayAnswer firing fireLatch");
@ -558,7 +558,7 @@ protected Object passThrough(InvocationOnMock invocation) throws Throwable {
resultLatch.countDown();
}
}
/**
* After calling proceed(), this will wait until the call has
* completed and a result has been returned to the caller.
@ -566,7 +566,7 @@ protected Object passThrough(InvocationOnMock invocation) throws Throwable {
public void waitForResult() throws InterruptedException {
resultLatch.await();
}
/**
* After the call has gone through, return any exception that
* was thrown, or null if no exception was thrown.
@ -574,7 +574,7 @@ public void waitForResult() throws InterruptedException {
public Throwable getThrown() {
return thrown;
}
/**
* After the call has gone through, return the call's return value,
* or null in case it was void or an exception was thrown.
@ -582,20 +582,20 @@ public Throwable getThrown() {
public Object getReturnValue() {
return returnValue;
}
public int getFireCount() {
return fireCounter.get();
}
public int getResultCount() {
return resultCounter.get();
}
}
/**
* An Answer implementation that simply forwards all calls through
* to a delegate.
*
*
* This is useful as the default Answer for a mock object, to create
* something like a spy on an RPC proxy. For example:
* <code>
@ -606,14 +606,14 @@ public int getResultCount() {
* ...
* </code>
*/
public static class DelegateAnswer implements Answer<Object> {
public static class DelegateAnswer implements Answer<Object> {
private final Object delegate;
private final Log log;
public DelegateAnswer(Object delegate) {
this(null, delegate);
}
public DelegateAnswer(Log log, Object delegate) {
this.log = log;
this.delegate = delegate;
@ -653,7 +653,7 @@ public SleepAnswer(int minSleepTime, int maxSleepTime) {
this.minSleepTime = minSleepTime;
this.maxSleepTime = maxSleepTime;
}
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
boolean interrupted = false;
@ -683,11 +683,11 @@ public static void assertMatches(String output, String pattern) {
" but got:\n" + output,
Pattern.compile(pattern).matcher(output).find());
}
public static void assertValueNear(long expected, long actual, long allowedError) {
assertValueWithinRange(expected - allowedError, expected + allowedError, actual);
}
public static void assertValueWithinRange(long expectedMin, long expectedMax,
long actual) {
Assert.assertTrue("Expected " + actual + " to be in range (" + expectedMin + ","
@ -842,4 +842,28 @@ public static void failif(boolean condition,
failf(format, args);
}
}
/**
* Retreive the max number of parallel test threads when running under maven.
* @return int number of threads
*/
public static int getTestsThreadCount() {
String propString = System.getProperty("testsThreadCount", "1");
int threadCount = 1;
if (propString != null) {
String trimProp = propString.trim();
if (trimProp.endsWith("C")) {
double multiplier = Double.parseDouble(
trimProp.substring(0, trimProp.length()-1));
double calculated = multiplier * ((double) Runtime
.getRuntime()
.availableProcessors());
threadCount = calculated > 0d ? Math.max((int) calculated, 1) : 0;
} else {
threadCount = Integer.parseInt(trimProp);
}
}
return threadCount;
}
}

26
hadoop-hdfs-project/hadoop-hdfs/pom.xml
View File

@ -515,30 +515,13 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<build>
<plugins>
<plugin>
<artifactId>maven-antrun-plugin</artifactId>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-maven-plugins</artifactId>
<executions>
<execution>
<id>create-parallel-tests-dirs</id>
<phase>test-compile</phase>
<configuration>
<target>
<script language="javascript"><![CDATA[
var baseDirs = [
"${test.build.data}",
"${test.build.dir}",
"${hadoop.tmp.dir}" ];
for (var i in baseDirs) {
for (var j = 1; j <= ${testsThreadCount}; ++j) {
var mkdir = project.createTask("mkdir");
mkdir.setDir(new java.io.File(baseDirs[i], j));
mkdir.perform();
}
}
]]></script>
</target>
</configuration>
<id>parallel-tests-createdir</id>
<goals>
<goal>run</goal>
<goal>parallel-tests-createdir</goal>
</goals>
</execution>
</executions>
@ -551,6 +534,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<reuseForks>false</reuseForks>
<argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
<systemPropertyVariables>
<testsThreadCount>${testsThreadCount}</testsThreadCount>
<test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
<test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
<hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>

100
hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/paralleltests/CreateDirsMojo.java Normal file
View File

@ -0,0 +1,100 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.maven.plugin.paralleltests;
import java.io.File;
import org.apache.maven.plugin.AbstractMojo;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugins.annotations.LifecyclePhase;
import org.apache.maven.plugins.annotations.Mojo;
import org.apache.maven.plugins.annotations.Parameter;
/**
* Goal which creates the parallel-test directories.
*/
@Mojo(name="parallel-tests-createdir",
defaultPhase = LifecyclePhase.GENERATE_TEST_RESOURCES)
public class CreateDirsMojo extends AbstractMojo {
/**
* Location of the test.build.dir.
*/
@Parameter(defaultValue="${project.build.directory}/test-dir")
private File testBuildDir;
/**
* Location of the test.build.data.
*/
@Parameter(defaultValue="${project.build.directory}/test-dir")
private File testBuildData;
/**
* Location of the test.build.data.
*/
@Parameter(defaultValue="${project.build.directory}/tmp")
private File hadoopTmpDir;
/**
* Thread count.
*/
@Parameter(defaultValue="${testsThreadCount}")
private String testsThreadCount;
public void execute() throws MojoExecutionException {
int numDirs=getTestsThreadCount();
mkParallelDirs(testBuildDir, numDirs);
mkParallelDirs(testBuildData, numDirs);
mkParallelDirs(hadoopTmpDir, numDirs);
}
/**
* Get the real number of parallel threads.
* @return int number of threads
*/
public int getTestsThreadCount() {
int threadCount = 1;
if (testsThreadCount != null) {
String trimProp = testsThreadCount.trim();
if (trimProp.endsWith("C")) {
double multiplier = Double.parseDouble(
trimProp.substring(0, trimProp.length()-1));
double calculated = multiplier * ((double) Runtime
.getRuntime()
.availableProcessors());
threadCount = calculated > 0d ? Math.max((int) calculated, 1) : 0;
} else {
threadCount = Integer.parseInt(testsThreadCount);
}
}
return threadCount;
}
private void mkParallelDirs(File testDir, int numDirs)
throws MojoExecutionException {
for (int i=1; i<=numDirs; i++) {
File newDir = new File(testDir, String.valueOf(i));
if (!newDir.exists()) {
getLog().info("Creating " + newDir.toString());
if (!newDir.mkdirs()) {
throw new MojoExecutionException("Unable to create "
+ newDir.toString());
}
}
}
}
}

26
hadoop-tools/hadoop-aws/pom.xml
View File

@ -85,30 +85,13 @@
<build>
<plugins>
<plugin>
<artifactId>maven-antrun-plugin</artifactId>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-maven-plugins</artifactId>
<executions>
<execution>
<id>create-parallel-tests-dirs</id>
<phase>test-compile</phase>
<configuration>
<target>
<script language="javascript"><![CDATA[
var baseDirs = [
"${test.build.data}",
"${test.build.dir}",
"${hadoop.tmp.dir}" ];
for (var i in baseDirs) {
for (var j = 1; j <= ${testsThreadCount}; ++j) {
var mkdir = project.createTask("mkdir");
mkdir.setDir(new java.io.File(baseDirs[i], j));
mkdir.perform();
}
}
]]></script>
</target>
</configuration>
<id>parallel-tests-createdir</id>
<goals>
<goal>run</goal>
<goal>parallel-tests-createdir</goal>
</goals>
</execution>
</executions>
@ -121,6 +104,7 @@
<reuseForks>false</reuseForks>
<argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
<systemPropertyVariables>
<testsThreadCount>${testsThreadCount}</testsThreadCount>
<test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
<test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
<hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>