HDFS-14130. [SBN read] Make ZKFC ObserverNode aware. Contributed by xiangheng and Konstantin Shvachko.

This commit is contained in:
Konstantin V Shvachko 2019-02-25 13:54:09 -08:00
parent 9de34d2990
commit a6ab37192a
3 changed files with 70 additions and 3 deletions

View File

@ -728,9 +728,9 @@ private ZKFCProtocol cedeRemoteActive(HAServiceTarget remote, int timeout)
}
/**
* Ensure that the local node is in a healthy state, and thus
* eligible for graceful failover.
* @throws ServiceFailedException if the node is unhealthy
* If the local node is an observer or is unhealthy it
* is not eligible for graceful failover.
* @throws ServiceFailedException if the node is an observer or unhealthy
*/
private synchronized void checkEligibleForFailover()
throws ServiceFailedException {
@ -740,6 +740,11 @@ private synchronized void checkEligibleForFailover()
localTarget + " is not currently healthy. " +
"Cannot be failover target");
}
if (serviceState == HAServiceState.OBSERVER) {
throw new ServiceFailedException(
localTarget + " is in observer state. " +
"Cannot be failover target");
}
}
/**
@ -856,6 +861,11 @@ void verifyChangedServiceState(HAServiceState changedState) {
}
return;
}
if (changedState == HAServiceState.OBSERVER) {
elector.quitElection(true);
serviceState = HAServiceState.OBSERVER;
return;
}
if (changedState == serviceState) {
serviceStateMismatchCount = 0;
return;

View File

@ -280,6 +280,21 @@ public void testAutoFailoverOnLostZKSession() throws Exception {
cluster.expireAndVerifyFailover(1, 0);
}
/**
* Test that the local node is observer.
*/
@Test
public void testVerifyObserverState()
throws Exception {
cluster.start(3);
DummyHAService svc2 = cluster.getService(2);
svc2.state = HAServiceState.OBSERVER;
// Verify svc2 is observer
LOG.info("Waiting for svc2 to enter observer state");
cluster.waitForHAState(2, HAServiceState.OBSERVER);
}
/**
* Test that, if the standby node is unhealthy, it doesn't try to become
* active
@ -474,6 +489,32 @@ public void testGracefulFailoverToUnhealthy() throws Exception {
}
}
@Test
public void testObserverExitGracefulFailover() throws Exception {
cluster.start(3);
cluster.waitForActiveLockHolder(0);
// Mark it become observer, wait for it to exit election
DummyHAService svc2 = cluster.getService(2);
svc2.state = HAServiceState.OBSERVER;
cluster.waitForHAState(2, HAServiceState.OBSERVER);
cluster.setFailToBecomeActive(2, true);
cluster.setFailToBecomeStandby(2, true);
cluster.setFailToBecomeObserver(2, true);
cluster.waitForElectorState(2, ActiveStandbyElector.State.INIT);
// Ask for failover, it should fail, because it's observer
try {
cluster.getService(2).getZKFCProxy(conf, 5000).gracefulFailover();
fail("Did not fail to graceful failover to observer!");
} catch (ServiceFailedException sfe) {
GenericTestUtils.assertExceptionContains(
cluster.getService(2).toString() +
" is in observer state.", sfe);
}
}
@Test
public void testGracefulFailoverFailBecomingActive() throws Exception {
cluster.start();

View File

@ -20,6 +20,8 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.util.concurrent.TimeoutException;
import org.apache.hadoop.conf.Configuration;
@ -212,6 +214,20 @@ public void testManualFailoverWithDFSHAAdmin() throws Exception {
tool.run(new String[]{"-failover", "nn2", "nn1"}));
waitForHAState(0, HAServiceState.ACTIVE);
waitForHAState(1, HAServiceState.STANDBY);
// Answer "yes" to the prompt for --forcemanual
InputStream inOriginial = System.in;
System.setIn(new ByteArrayInputStream("yes\n".getBytes()));
int result = tool.run(
new String[]{"-transitionToObserver", "-forcemanual", "nn2"});
assertEquals("State transition returned: " + result, 0, result);
waitForHAState(1, HAServiceState.OBSERVER);
// Answer "yes" to the prompt for --forcemanual
System.setIn(new ByteArrayInputStream("yes\n".getBytes()));
result = tool.run(
new String[]{"-transitionToStandby", "-forcemanual", "nn2"});
System.setIn(inOriginial);
assertEquals("State transition returned: " + result, 0, result);
waitForHAState(1, HAServiceState.STANDBY);
}
private void waitForHAState(int nnidx, final HAServiceState state)