HDFS-3894. QJM: testRecoverAfterDoubleFailures can be flaky due to IPC client caching. Contributed by Todd Lipcon.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-3077@1384592 13f79535-47bb-0310-9956-ffa450edef68

hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-3077.txt

@@ -70,3 +70,5 @@ HDFS-3915. QJM: Failover fails with auth error in secure cluster (todd)
 HDFS-3906. QJM: quorum timeout on failover with large log segment (todd)
 
 HDFS-3840. JournalNodes log JournalNotFormattedException backtrace error before being formatted (todd)
+
+HDFS-3894. QJM: testRecoverAfterDoubleFailures can be flaky due to IPC client caching (todd)

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java

@@ -47,6 +47,7 @@
 import org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream;
 import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
 import org.apache.hadoop.hdfs.util.Holder;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.ipc.ProtobufRpcEngine;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.log4j.Level;
@@ -90,9 +91,10 @@ public class TestQJMWithFaults {
   private static long determineMaxIpcNumber() throws Exception {
     Configuration conf = new Configuration();
     MiniJournalCluster cluster = new MiniJournalCluster.Builder(conf).build();
+    QuorumJournalManager qjm = null;
     long ret;
     try {
-      QuorumJournalManager qjm = createInjectableQJM(cluster);
+      qjm = createInjectableQJM(cluster);
       qjm.format(FAKE_NSINFO);
       doWorkload(cluster, qjm);
       
@@ -110,6 +112,7 @@ private static long determineMaxIpcNumber() throws Exception {
       ret = ipcCounts.first();
       LOG.info("Max IPC count = " + ret);
     } finally {
+      IOUtils.closeStream(qjm);
       cluster.shutdown();
     }
     return ret;
@@ -136,8 +139,8 @@ public void testRecoverAfterDoubleFailures() throws Exception {
         
         MiniJournalCluster cluster = new MiniJournalCluster.Builder(conf)
           .build();
+        QuorumJournalManager qjm = null;
         try {
-          QuorumJournalManager qjm;
           qjm = createInjectableQJM(cluster);
           qjm.format(FAKE_NSINFO);
           List<AsyncLogger> loggers = qjm.getLoggerSetForTests().getLoggersForTests();
@@ -150,6 +153,8 @@ public void testRecoverAfterDoubleFailures() throws Exception {
                 ". This is expected since we injected a failure in the " +
                 "majority.");
           }
+          qjm.close();
+          qjm = null;
 
           // Now should be able to recover
           qjm = createInjectableQJM(cluster);
@@ -165,6 +170,8 @@ public void testRecoverAfterDoubleFailures() throws Exception {
         } finally {
           cluster.shutdown();
           cluster = null;
+          IOUtils.closeStream(qjm);
+          qjm = null;
         }
       }
     }
@@ -405,7 +412,6 @@ private static QJournalProtocol mockProxy(WrapEveryCall<Object> wrapper)
         Mockito.withSettings()
           .defaultAnswer(wrapper)
           .extraInterfaces(Closeable.class));
-    Mockito.doNothing().when((Closeable)mock).close();
     return mock;
   }
 
@@ -418,7 +424,12 @@ private static abstract class WrapEveryCall<T> implements Answer<T> {
     @SuppressWarnings("unchecked")
     @Override
     public T answer(InvocationOnMock invocation) throws Throwable {
-      beforeCall(invocation);
+      // Don't want to inject an error on close() since that isn't
+      // actually an IPC call!
+      if (!Closeable.class.equals(
+            invocation.getMethod().getDeclaringClass())) {
+        beforeCall(invocation);
+      }
 
       try {
         return (T) invocation.getMethod().invoke(realObj,