big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-5685. RM configuration allows all failover methods to disabled when automatic failover is enabled

Browse Source
This commit is contained in:
Daniel Templeton 2017-03-29 12:36:15 -07:00
parent 13c766b62c
commit 640ba1d23f
4 changed files with 191 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
View File

@ -34,6 +34,7 @@
public class HAUtil {
private static Log LOG = LogFactory.getLog(HAUtil.class);
@VisibleForTesting
public static final String BAD_CONFIG_MESSAGE_PREFIX =
"Invalid configuration! ";
@ -79,6 +80,7 @@ public static void verifyAndSetConfiguration(Configuration conf)
throws YarnRuntimeException {
verifyAndSetRMHAIdsList(conf);
verifyAndSetCurrentRMHAId(conf);
verifyLeaderElection(conf);
verifyAndSetAllServiceAddresses(conf);
}
@ -117,7 +119,7 @@ private static void verifyAndSetCurrentRMHAId(Configuration conf) {
msg.append("Can not find valid RM_HA_ID. None of ");
for (String id : conf
.getTrimmedStringCollection(YarnConfiguration.RM_HA_IDS)) {
msg.append(addSuffix(YarnConfiguration.RM_ADDRESS, id) + " ");
msg.append(addSuffix(YarnConfiguration.RM_ADDRESS, id)).append(" ");
}
msg.append(" are matching" +
" the local address OR " + YarnConfiguration.RM_HA_ID + " is not" +
@ -133,6 +135,32 @@ private static void verifyAndSetCurrentRMHAId(Configuration conf) {
conf.set(YarnConfiguration.RM_HA_ID, rmId);
}
/**
* This method validates that some leader election service is enabled. YARN
* allows leadership election to be disabled in the configuration, which
* breaks automatic failover. If leadership election is disabled, this
* method will throw an exception via
* {@link #throwBadConfigurationException(java.lang.String)}.
*
* @param conf the {@link Configuration} to validate
*/
private static void verifyLeaderElection(Configuration conf) {
if (isAutomaticFailoverEnabled(conf) &&
!conf.getBoolean(YarnConfiguration.CURATOR_LEADER_ELECTOR,
YarnConfiguration.DEFAULT_CURATOR_LEADER_ELECTOR_ENABLED) &&
!isAutomaticFailoverEmbedded(conf)) {
throwBadConfigurationException(NO_LEADER_ELECTION_MESSAGE);
}
}
@VisibleForTesting
static final String NO_LEADER_ELECTION_MESSAGE =
"The yarn.resourcemanager.ha.automatic-failover.embedded "
+ "and yarn.resourcemanager.ha.curator-leader-elector.enabled "
+ "properties are both false. One of these two properties must "
+ "be true when yarn.resourcemanager.ha.automatic-failover.enabled "
+ "is true";
private static void verifyAndSetConfValue(String prefix, Configuration conf) {
String confKey = null;
String confValue = null;

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -627,8 +627,22 @@ public static boolean isAclEnabled(Configuration conf) {
AUTO_FAILOVER_PREFIX + "enabled";
public static final boolean DEFAULT_AUTO_FAILOVER_ENABLED = true;
/**
* This property controls whether {@link ActiveStandbyElector} leader
* election should be used when {@link #CURATOR_LEADER_ELECTOR} is
* {@code false}.
*
* @deprecated This property should never be set to {@code false}.
*/
@Deprecated
public static final String AUTO_FAILOVER_EMBEDDED =
AUTO_FAILOVER_PREFIX + "embedded";
/**
* The default value for {@link #AUTO_FAILOVER_EMBEDDED}.
*
* @deprecated The {@link #AUTO_FAILOVER_EMBEDDED} property is deprecated.
*/
@Deprecated
public static final boolean DEFAULT_AUTO_FAILOVER_EMBEDDED = true;
public static final String AUTO_FAILOVER_ZK_BASE_PATH =
@ -667,7 +681,7 @@ public static boolean isAclEnabled(Configuration conf) {
/**
* Whether to use curator-based elector for leader election.
* Whether to use the Curator-based elector for leader election.
*
* @deprecated Eventually, we want to default to the curator-based
* implementation and remove the {@link ActiveStandbyElector} based

86
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java
View File

@ -85,44 +85,47 @@ public void testGetRMId() throws Exception {
@Test
public void testVerifyAndSetConfiguration() throws Exception {
Configuration myConf = new Configuration(conf);
try {
HAUtil.verifyAndSetConfiguration(conf);
HAUtil.verifyAndSetConfiguration(myConf);
} catch (YarnRuntimeException e) {
fail("Should not throw any exceptions.");
}
assertEquals("Should be saved as Trimmed collection",
StringUtils.getStringCollection(RM_NODE_IDS), HAUtil.getRMHAIds(conf));
StringUtils.getStringCollection(RM_NODE_IDS),
HAUtil.getRMHAIds(myConf));
assertEquals("Should be saved as Trimmed string",
RM1_NODE_ID, HAUtil.getRMHAId(conf));
for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) {
RM1_NODE_ID, HAUtil.getRMHAId(myConf));
for (String confKey : YarnConfiguration.getServiceAddressConfKeys(myConf)) {
assertEquals("RPC address not set for " + confKey,
RM1_ADDRESS, conf.get(confKey));
RM1_ADDRESS, myConf.get(confKey));
}
conf.clear();
conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID);
myConf = new Configuration(conf);
myConf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID);
try {
HAUtil.verifyAndSetConfiguration(conf);
HAUtil.verifyAndSetConfiguration(myConf);
} catch (YarnRuntimeException e) {
assertEquals("YarnRuntimeException by verifyAndSetRMHAIds()",
HAUtil.BAD_CONFIG_MESSAGE_PREFIX +
HAUtil.getInvalidValueMessage(YarnConfiguration.RM_HA_IDS,
conf.get(YarnConfiguration.RM_HA_IDS) +
myConf.get(YarnConfiguration.RM_HA_IDS) +
"\nHA mode requires atleast two RMs"),
e.getMessage());
}
conf.clear();
myConf = new Configuration(conf);
// simulate the case YarnConfiguration.RM_HA_ID is not set
conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + ","
myConf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + ","
+ RM2_NODE_ID);
for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) {
conf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS);
conf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS);
for (String confKey : YarnConfiguration.getServiceAddressConfKeys(myConf)) {
myConf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS);
myConf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS);
}
try {
HAUtil.verifyAndSetConfiguration(conf);
HAUtil.verifyAndSetConfiguration(myConf);
} catch (YarnRuntimeException e) {
assertEquals("YarnRuntimeException by getRMId()",
HAUtil.BAD_CONFIG_MESSAGE_PREFIX +
@ -130,16 +133,16 @@ public void testVerifyAndSetConfiguration() throws Exception {
e.getMessage());
}
conf.clear();
conf.set(YarnConfiguration.RM_HA_ID, RM_INVALID_NODE_ID);
conf.set(YarnConfiguration.RM_HA_IDS, RM_INVALID_NODE_ID + ","
myConf = new Configuration(conf);
myConf.set(YarnConfiguration.RM_HA_ID, RM_INVALID_NODE_ID);
myConf.set(YarnConfiguration.RM_HA_IDS, RM_INVALID_NODE_ID + ","
+ RM1_NODE_ID);
for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) {
for (String confKey : YarnConfiguration.getServiceAddressConfKeys(myConf)) {
// simulate xml with invalid node id
conf.set(confKey + RM_INVALID_NODE_ID, RM_INVALID_NODE_ID);
myConf.set(confKey + RM_INVALID_NODE_ID, RM_INVALID_NODE_ID);
}
try {
HAUtil.verifyAndSetConfiguration(conf);
HAUtil.verifyAndSetConfiguration(myConf);
} catch (YarnRuntimeException e) {
assertEquals("YarnRuntimeException by addSuffix()",
HAUtil.BAD_CONFIG_MESSAGE_PREFIX +
@ -148,12 +151,12 @@ public void testVerifyAndSetConfiguration() throws Exception {
e.getMessage());
}
conf.clear();
myConf = new Configuration();
// simulate the case HAUtil.RM_RPC_ADDRESS_CONF_KEYS are not set
conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID);
conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
myConf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID);
myConf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
try {
HAUtil.verifyAndSetConfiguration(conf);
HAUtil.verifyAndSetConfiguration(myConf);
fail("Should throw YarnRuntimeException. by Configuration#set()");
} catch (YarnRuntimeException e) {
String confKey =
@ -166,21 +169,36 @@ public void testVerifyAndSetConfiguration() throws Exception {
// simulate the case YarnConfiguration.RM_HA_IDS doesn't contain
// the value of YarnConfiguration.RM_HA_ID
conf.clear();
conf.set(YarnConfiguration.RM_HA_IDS, RM2_NODE_ID + "," + RM3_NODE_ID);
conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID_UNTRIMMED);
for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) {
conf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS_UNTRIMMED);
conf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS);
conf.set(HAUtil.addSuffix(confKey, RM3_NODE_ID), RM3_ADDRESS);
myConf = new Configuration(conf);
myConf.set(YarnConfiguration.RM_HA_IDS, RM2_NODE_ID + "," + RM3_NODE_ID);
myConf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID_UNTRIMMED);
for (String confKey : YarnConfiguration.getServiceAddressConfKeys(myConf)) {
myConf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS_UNTRIMMED);
myConf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS);
myConf.set(HAUtil.addSuffix(confKey, RM3_NODE_ID), RM3_ADDRESS);
}
try {
HAUtil.verifyAndSetConfiguration(conf);
HAUtil.verifyAndSetConfiguration(myConf);
} catch (YarnRuntimeException e) {
assertEquals("YarnRuntimeException by getRMId()'s validation",
HAUtil.BAD_CONFIG_MESSAGE_PREFIX +
HAUtil.getRMHAIdNeedToBeIncludedMessage("[rm2, rm3]", RM1_NODE_ID),
e.getMessage());
e.getMessage());
}
// simulate the case that no leader election is enabled
myConf = new Configuration(conf);
myConf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
myConf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, true);
myConf.setBoolean(YarnConfiguration.AUTO_FAILOVER_EMBEDDED, false);
myConf.setBoolean(YarnConfiguration.CURATOR_LEADER_ELECTOR, false);
try {
HAUtil.verifyAndSetConfiguration(myConf);
} catch (YarnRuntimeException e) {
assertEquals("YarnRuntimeException by getRMId()'s validation",
HAUtil.BAD_CONFIG_MESSAGE_PREFIX + HAUtil.NO_LEADER_ELECTION_MESSAGE,
e.getMessage());
}
}

95
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
View File

@ -78,6 +78,8 @@
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import static org.junit.Assert.assertTrue;
public class TestRMAdminService {
@ -841,6 +843,78 @@ public void testRMHAWithFileSystemBasedConfiguration() throws IOException,
}
}
/**
* Test that a configuration with no leader election configured fails.
*/
@Test
public void testHAConfWithoutLeaderElection() {
Configuration conf = new Configuration(configuration);
conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, true);
conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_EMBEDDED, false);
conf.setBoolean(YarnConfiguration.CURATOR_LEADER_ELECTOR, false);
conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2");
int base = 100;
for (String confKey :
YarnConfiguration.getServiceAddressConfKeys(configuration)) {
conf.set(HAUtil.addSuffix(confKey, "rm1"), "0.0.0.0:"
+ (base + 20));
conf.set(HAUtil.addSuffix(confKey, "rm2"), "0.0.0.0:"
+ (base + 40));
base = base * 2;
}
conf.set(YarnConfiguration.RM_HA_ID, "rm1");
checkBadConfiguration(conf);
}
/**
* Test that a configuration with a single RM fails.
*/
@Test
public void testHAConfWithSingleRMID() {
Configuration conf = new Configuration(configuration);
conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, true);
conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_EMBEDDED, true);
conf.set(YarnConfiguration.RM_HA_IDS, "rm1");
int base = 100;
for (String confKey :
YarnConfiguration.getServiceAddressConfKeys(configuration)) {
conf.set(HAUtil.addSuffix(confKey, "rm1"), "0.0.0.0:"
+ (base + 20));
base = base * 2;
}
conf.set(YarnConfiguration.RM_HA_ID, "rm1");
checkBadConfiguration(conf);
}
/**
* Test that a configuration with no service information fails.
*/
@Test
public void testHAConfWithoutServiceInfo() {
Configuration conf = new Configuration(configuration);
conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, true);
conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_EMBEDDED, true);
conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2");
conf.set(YarnConfiguration.RM_HA_ID, "rm1");
checkBadConfiguration(conf);
}
@Test
public void testRMStartsWithoutConfigurationFilesProvided() {
// enable FileSystemBasedConfigurationProvider without uploading
@ -1368,4 +1442,25 @@ private void updateConfigurationForRMHA() {
base = base * 2;
}
}
/**
* This method initializes an RM with the given configuration and expects it
* to fail with a configuration error.
*
* @param conf the {@link Configuration} to use
*/
private void checkBadConfiguration(Configuration conf) {
MockRM rm1 = null;
conf.set(YarnConfiguration.RM_HA_ID, "rm1");
try {
rm1 = new MockRM(conf);
rm1.init(conf);
fail("The RM allowed an invalid configuration");
} catch (YarnRuntimeException e) {
assertTrue("The RM initialization threw an unexpected exception",
e.getMessage().startsWith(HAUtil.BAD_CONFIG_MESSAGE_PREFIX));
}
}
}