big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-12706. TestLocalFsFCStatistics#testStatisticsThreadLocalDataCleanUp times out occasionally. Contributed by Sangjin Lee and Colin Patrick McCabe

Browse Source
This commit is contained in:
Jason Lowe 2016-01-14 23:45:03 +00:00
parent 651c23e8ef
commit cdf8895259
2 changed files with 22 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -1652,6 +1652,9 @@ Release 2.7.3 - UNRELEASED
HADOOP-12107. long running apps may have a huge number of StatisticsData
instances under FileSystem (Sangjin Lee via Ming Ma)
HADOOP-12706. TestLocalFsFCStatistics#testStatisticsThreadLocalDataCleanUp
times out occasionally (Sangjin Lee and Colin Patrick McCabe via jlowe)
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
@ -2449,6 +2452,9 @@ Release 2.6.4 - UNRELEASED
HADOOP-12107. long running apps may have a huge number of StatisticsData
instances under FileSystem (Sangjin Lee via Ming Ma)
HADOOP-12706. TestLocalFsFCStatistics#testStatisticsThreadLocalDataCleanUp
times out occasionally (Sangjin Lee and Colin Patrick McCabe via jlowe)
Release 2.6.3 - 2015-12-17
INCOMPATIBLE CHANGES

21
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
View File

@ -32,6 +32,8 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileSystem.Statistics;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
@ -46,6 +48,8 @@
* </p>
*/
public abstract class FCStatisticsBaseTest {
private static final Log LOG = LogFactory.getLog(FCStatisticsBaseTest.class);
static protected int blockSize = 512;
static protected int numBlocks = 1;
@ -110,7 +114,7 @@ public void testStatistics() throws IOException, URISyntaxException {
fc.delete(filePath, true);
}
@Test(timeout=60000)
@Test(timeout=70000)
public void testStatisticsThreadLocalDataCleanUp() throws Exception {
final Statistics stats = new Statistics("test");
// create a small thread pool to test the statistics
@ -137,17 +141,24 @@ public Boolean call() {
es.shutdownNow();
es.awaitTermination(1, TimeUnit.MINUTES);
es = null;
System.gc();
System.gc(); // force GC to garbage collect threads
// wait for up to 10 seconds
// wait for up to 60 seconds
GenericTestUtils.waitFor(new Supplier<Boolean>() {
@Override
public Boolean get() {
int size = stats.getAllThreadLocalDataSize();
allDataSize.set(size);
return size == 0;
if (size == 0) {
return true;
}
}, 1000, 10*1000);
LOG.warn("not all references have been cleaned up; still " +
allDataSize.get() + " references left");
LOG.warn("triggering another GC");
System.gc();
return false;
}
}, 500, 60*1000);
Assert.assertEquals(0, allDataSize.get());
Assert.assertEquals(size, stats.getReadOps());
}