big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-3894. QJM: testRecoverAfterDoubleFailures can be flaky due to IPC client caching. Contributed by Todd Lipcon.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-3077@1384592 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Todd Lipcon 2012-09-13 23:00:22 +00:00
parent a93ba1648a
commit 40d56f1741
2 changed files with 17 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-3077.txt
View File

@ -70,3 +70,5 @@ HDFS-3915. QJM: Failover fails with auth error in secure cluster (todd)
HDFS-3906. QJM: quorum timeout on failover with large log segment (todd) HDFS-3906. QJM: quorum timeout on failover with large log segment (todd)
HDFS-3840. JournalNodes log JournalNotFormattedException backtrace error before being formatted (todd) HDFS-3840. JournalNodes log JournalNotFormattedException backtrace error before being formatted (todd)
HDFS-3894. QJM: testRecoverAfterDoubleFailures can be flaky due to IPC client caching (todd)

17
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQJMWithFaults.java
View File

@ -47,6 +47,7 @@
import org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream; import org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.util.Holder; import org.apache.hadoop.hdfs.util.Holder;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.ProtobufRpcEngine; import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.GenericTestUtils;
import org.apache.log4j.Level; import org.apache.log4j.Level;
@ -90,9 +91,10 @@ public class TestQJMWithFaults {
private static long determineMaxIpcNumber() throws Exception { private static long determineMaxIpcNumber() throws Exception {
Configuration conf = new Configuration(); Configuration conf = new Configuration();
MiniJournalCluster cluster = new MiniJournalCluster.Builder(conf).build(); MiniJournalCluster cluster = new MiniJournalCluster.Builder(conf).build();
QuorumJournalManager qjm = null;
long ret; long ret;
try { try {
QuorumJournalManager qjm = createInjectableQJM(cluster); qjm = createInjectableQJM(cluster);
qjm.format(FAKE_NSINFO); qjm.format(FAKE_NSINFO);
doWorkload(cluster, qjm); doWorkload(cluster, qjm);
@ -110,6 +112,7 @@ private static long determineMaxIpcNumber() throws Exception {
ret = ipcCounts.first(); ret = ipcCounts.first();
LOG.info("Max IPC count = " + ret); LOG.info("Max IPC count = " + ret);
} finally { } finally {
IOUtils.closeStream(qjm);
cluster.shutdown(); cluster.shutdown();
} }
return ret; return ret;
@ -136,8 +139,8 @@ public void testRecoverAfterDoubleFailures() throws Exception {
MiniJournalCluster cluster = new MiniJournalCluster.Builder(conf) MiniJournalCluster cluster = new MiniJournalCluster.Builder(conf)
.build(); .build();
QuorumJournalManager qjm = null;
try { try {
QuorumJournalManager qjm;
qjm = createInjectableQJM(cluster); qjm = createInjectableQJM(cluster);
qjm.format(FAKE_NSINFO); qjm.format(FAKE_NSINFO);
List<AsyncLogger> loggers = qjm.getLoggerSetForTests().getLoggersForTests(); List<AsyncLogger> loggers = qjm.getLoggerSetForTests().getLoggersForTests();
@ -150,6 +153,8 @@ public void testRecoverAfterDoubleFailures() throws Exception {
". This is expected since we injected a failure in the " + ". This is expected since we injected a failure in the " +
"majority."); "majority.");
} }
qjm.close();
qjm = null;
// Now should be able to recover // Now should be able to recover
qjm = createInjectableQJM(cluster); qjm = createInjectableQJM(cluster);
@ -165,6 +170,8 @@ public void testRecoverAfterDoubleFailures() throws Exception {
} finally { } finally {
cluster.shutdown(); cluster.shutdown();
cluster = null; cluster = null;
IOUtils.closeStream(qjm);
qjm = null;
} }
} }
} }
@ -405,7 +412,6 @@ private static QJournalProtocol mockProxy(WrapEveryCall<Object> wrapper)
Mockito.withSettings() Mockito.withSettings()
.defaultAnswer(wrapper) .defaultAnswer(wrapper)
.extraInterfaces(Closeable.class)); .extraInterfaces(Closeable.class));
Mockito.doNothing().when((Closeable)mock).close();
return mock; return mock;
} }
@ -418,7 +424,12 @@ private static abstract class WrapEveryCall<T> implements Answer<T> {
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
@Override @Override
public T answer(InvocationOnMock invocation) throws Throwable { public T answer(InvocationOnMock invocation) throws Throwable {
// Don't want to inject an error on close() since that isn't
// actually an IPC call!
if (!Closeable.class.equals(
invocation.getMethod().getDeclaringClass())) {
beforeCall(invocation); beforeCall(invocation);
}
try { try {
return (T) invocation.getMethod().invoke(realObj, return (T) invocation.getMethod().invoke(realObj,