big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-15098. TestClusterTopology#testChooseRandom fails intermittently. Contributed by Zsolt Venczel.

Browse Source
This commit is contained in:
Sean Mackrory 2017-12-07 10:52:02 -07:00
parent 67662e2ac9
commit acb92904d0
1 changed files with 35 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestClusterTopology.java
View File

@ -139,38 +139,50 @@ public void testChooseRandom() {
NodeElement node4 = getNewNode("node4", "/d1/r3"); NodeElement node4 = getNewNode("node4", "/d1/r3");
cluster.add(node4); cluster.add(node4);
// Number of test runs
int numTestRuns = 3;
int chiSquareTestRejectedCounter = 0;
// Number of iterations to do the test // Number of iterations to do the test
int numIterations = 100; int numIterations = 100;
// Pick random nodes for (int testRun = 0; testRun < numTestRuns; ++testRun) {
HashMap<String,Integer> histogram = new HashMap<String,Integer>();
for (int i=0; i<numIterations; i++) {
String randomNode = cluster.chooseRandom(NodeBase.ROOT).getName();
if (!histogram.containsKey(randomNode)) {
histogram.put(randomNode, 0);
}
histogram.put(randomNode, histogram.get(randomNode) + 1);
}
assertEquals("Random is not selecting all nodes", 4, histogram.size());
// Check with 99% confidence (alpha=0.01 as confidence = (100 * (1 - alpha) // Pick random nodes
ChiSquareTest chiSquareTest = new ChiSquareTest(); HashMap<String, Integer> histogram = new HashMap<String, Integer>();
double[] expected = new double[histogram.size()]; for (int i = 0; i < numIterations; i++) {
long[] observed = new long[histogram.size()]; String randomNode = cluster.chooseRandom(NodeBase.ROOT).getName();
int j=0; if (!histogram.containsKey(randomNode)) {
for (Integer occurrence : histogram.values()) { histogram.put(randomNode, 0);
expected[j] = 1.0 * numIterations / histogram.size(); }
observed[j] = occurrence; histogram.put(randomNode, histogram.get(randomNode) + 1);
j++; }
assertEquals("Random is not selecting all nodes", 4, histogram.size());
// Check with 99% confidence alpha=0.01 as confidence = 100 * (1 - alpha)
ChiSquareTest chiSquareTest = new ChiSquareTest();
double[] expected = new double[histogram.size()];
long[] observed = new long[histogram.size()];
int j = 0;
for (Integer occurrence : histogram.values()) {
expected[j] = 1.0 * numIterations / histogram.size();
observed[j] = occurrence;
j++;
}
boolean chiSquareTestRejected =
chiSquareTest.chiSquareTest(expected, observed, 0.01);
if (chiSquareTestRejected) {
++chiSquareTestRejectedCounter;
}
} }
boolean chiSquareTestRejected =
chiSquareTest.chiSquareTest(expected, observed, 0.01);
// Check that they have the proper distribution // Check that they have the proper distribution
assertFalse("Not choosing nodes randomly", chiSquareTestRejected); assertFalse("Random not choosing nodes with proper distribution",
chiSquareTestRejectedCounter==3);
// Pick random nodes excluding the 2 nodes in /d1/r3 // Pick random nodes excluding the 2 nodes in /d1/r3
histogram = new HashMap<String,Integer>(); HashMap<String, Integer> histogram = new HashMap<String, Integer>();
for (int i=0; i<numIterations; i++) { for (int i=0; i<numIterations; i++) {
String randomNode = cluster.chooseRandom("~/d1/r3").getName(); String randomNode = cluster.chooseRandom("~/d1/r3").getName();
if (!histogram.containsKey(randomNode)) { if (!histogram.containsKey(randomNode)) {