big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-13819. TestDirectoryScanner#testDirectoryScannerInFederatedCluster is flaky

Browse Source 
Change-Id: I1cea6e67fcec72702ad202775dee3373261ac5cd
This commit is contained in:
Daniel Templeton 2018-08-14 17:03:10 -07:00
parent 363bd16e31
commit 4a5006b1d0
1 changed files with 32 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java
View File

@ -40,6 +40,7 @@
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.io.FileUtils;
@ -312,18 +313,29 @@ private long createBlockMetaFile() throws IOException {
return id;
}
private void scan(long totalBlocks, int diffsize, long missingMetaFile, long missingBlockFile,
long missingMemoryBlocks, long mismatchBlocks) throws IOException {
private void scan(long totalBlocks, int diffsize, long missingMetaFile,
long missingBlockFile, long missingMemoryBlocks, long mismatchBlocks)
throws IOException, InterruptedException, TimeoutException {
scan(totalBlocks, diffsize, missingMetaFile, missingBlockFile,
missingMemoryBlocks, mismatchBlocks, 0);
}
private void scan(long totalBlocks, int diffsize, long missingMetaFile,
long missingBlockFile, long missingMemoryBlocks, long mismatchBlocks,
long duplicateBlocks) throws IOException {
long duplicateBlocks)
throws IOException, InterruptedException, TimeoutException {
scanner.reconcile();
verifyStats(totalBlocks, diffsize, missingMetaFile, missingBlockFile,
missingMemoryBlocks, mismatchBlocks, duplicateBlocks);
GenericTestUtils.waitFor(() -> {
try {
verifyStats(totalBlocks, diffsize, missingMetaFile, missingBlockFile,
missingMemoryBlocks, mismatchBlocks, duplicateBlocks);
} catch (AssertionError ex) {
return false;
}
return true;
}, 50, 2000);
}
private void verifyStats(long totalBlocks, int diffsize, long missingMetaFile,
@ -785,7 +797,8 @@ public void run() {
}
}
private float runThrottleTest(int blocks) throws IOException {
private float runThrottleTest(int blocks)
throws IOException, InterruptedException, TimeoutException {
scanner.setRetainDiffs(true);
scan(blocks, 0, 0, 0, 0, 0);
scanner.shutdown();
@ -1069,10 +1082,19 @@ public void testDirectoryScannerInFederatedCluster() throws Exception {
scanner.setRetainDiffs(true);
scanner.reconcile();
//Check blocks in corresponding BP
bpid = cluster.getNamesystem(1).getBlockPoolId();
verifyStats(bp1Files, 0, 0, 0, 0, 0, 0);
bpid = cluster.getNamesystem(3).getBlockPoolId();
verifyStats(bp2Files, 0, 0, 0, 0, 0, 0);
GenericTestUtils.waitFor(() -> {
try {
bpid = cluster.getNamesystem(1).getBlockPoolId();
verifyStats(bp1Files, 0, 0, 0, 0, 0, 0);
bpid = cluster.getNamesystem(3).getBlockPoolId();
verifyStats(bp2Files, 0, 0, 0, 0, 0, 0);
} catch (AssertionError ex) {
return false;
}
return true;
}, 50, 2000);
} finally {
if (scanner != null) {
scanner.shutdown();