diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java index 02e9faac70..9c14bb43ce 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java @@ -417,6 +417,26 @@ public static void assertValueWithinRange(long expectedMin, long expectedMax, + expectedMax + ")", expectedMin <= actual && actual <= expectedMax); } + /** + * Determine if there are any threads whose name matches the regex. + * @param pattern a Pattern object used to match thread names + * @return true if there is any thread that matches the pattern + */ + public static boolean anyThreadMatching(Pattern pattern) { + ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); + + ThreadInfo[] infos = + threadBean.getThreadInfo(threadBean.getAllThreadIds(), 20); + for (ThreadInfo info : infos) { + if (info == null) + continue; + if (pattern.matcher(info.getThreadName()).matches()) { + return true; + } + } + return false; + } + /** * Assert that there are no threads running whose name matches the * given regular expression. @@ -424,18 +444,34 @@ public static void assertValueWithinRange(long expectedMin, long expectedMax, */ public static void assertNoThreadsMatching(String regex) { Pattern pattern = Pattern.compile(regex); - ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); - - ThreadInfo[] infos = threadBean.getThreadInfo(threadBean.getAllThreadIds(), 20); - for (ThreadInfo info : infos) { - if (info == null) continue; - if (pattern.matcher(info.getThreadName()).matches()) { - Assert.fail("Leaked thread: " + info + "\n" + - Joiner.on("\n").join(info.getStackTrace())); - } + if (anyThreadMatching(pattern)) { + Assert.fail("Leaked thread matches " + regex); } } + /** + * Periodically check and wait for any threads whose name match the + * given regular expression. + * + * @param regex the regex to match against. + * @param checkEveryMillis time (in milliseconds) between checks. + * @param waitForMillis total time (in milliseconds) to wait before throwing + * a time out exception. + * @throws TimeoutException + * @throws InterruptedException + */ + public static void waitForThreadTermination(String regex, + int checkEveryMillis, final int waitForMillis) throws TimeoutException, + InterruptedException { + final Pattern pattern = Pattern.compile(regex); + waitFor(new Supplier() { + @Override public Boolean get() { + return !anyThreadMatching(pattern); + } + }, checkEveryMillis, waitForMillis); + } + + /** * Skip test if native build profile of Maven is not activated. * Sub-project using this must set 'runningWithNative' property to true diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 2286203d62..920aab4328 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -2532,6 +2532,9 @@ Release 2.8.0 - UNRELEASED replicas should go through BlockPlacementPolicy (J.Andreina via vinayakumarb) + HDFS-9347. Invariant assumption in TestQuorumJournalManager.shutdown() + is wrong. (Wei-Chiu Chuang via zhz) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java index 3ba573c919..7d770e067a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java @@ -38,6 +38,7 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeoutException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -103,21 +104,23 @@ public void setup() throws Exception { qjm.recoverUnfinalizedSegments(); assertEquals(1, qjm.getLoggerSetForTests().getEpoch()); } - + @After - public void shutdown() throws IOException { + public void shutdown() throws IOException, InterruptedException, + TimeoutException { IOUtils.cleanup(LOG, toClose.toArray(new Closeable[0])); - + // Should not leak clients between tests -- this can cause flaky tests. // (See HDFS-4643) - GenericTestUtils.assertNoThreadsMatching(".*IPC Client.*"); + // Wait for IPC clients to terminate to avoid flaky tests + GenericTestUtils.waitForThreadTermination(".*IPC Client.*", 100, 1000); if (cluster != null) { cluster.shutdown(); cluster = null; } } - + /** * Enqueue a QJM for closing during shutdown. This makes the code a little * easier to follow, with fewer try..finally clauses necessary.