HDFS-2885. Remove "federation" from the nameservice config options. Contributed by Tsz Wo (Nicholas), SZE

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1340841 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Eli Collins 2012-05-20 21:13:17 +00:00
parent 75041b9950
commit 260dfa367e
24 changed files with 65 additions and 61 deletions

View File

@ -193,6 +193,9 @@ Release 2.0.1-alpha - UNRELEASED
HDFS-3438. BootstrapStandby should not require a rollEdits on active node HDFS-3438. BootstrapStandby should not require a rollEdits on active node
(todd) (todd)
HDFS-2885. Remove "federation" from the nameservice config options.
(Tsz Wo (Nicholas) via eli)
OPTIMIZATIONS OPTIMIZATIONS
BUG FIXES BUG FIXES

View File

@ -324,8 +324,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_NAMENODE_NAME_CACHE_THRESHOLD_KEY = "dfs.namenode.name.cache.threshold"; public static final String DFS_NAMENODE_NAME_CACHE_THRESHOLD_KEY = "dfs.namenode.name.cache.threshold";
public static final int DFS_NAMENODE_NAME_CACHE_THRESHOLD_DEFAULT = 10; public static final int DFS_NAMENODE_NAME_CACHE_THRESHOLD_DEFAULT = 10;
public static final String DFS_FEDERATION_NAMESERVICES = "dfs.federation.nameservices"; public static final String DFS_NAMESERVICES = "dfs.nameservices";
public static final String DFS_FEDERATION_NAMESERVICE_ID = "dfs.federation.nameservice.id"; public static final String DFS_NAMESERVICE_ID = "dfs.nameservice.id";
public static final String DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY = "dfs.namenode.resource.check.interval"; public static final String DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY = "dfs.namenode.resource.check.interval";
public static final int DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_DEFAULT = 5000; public static final int DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_DEFAULT = 5000;
public static final String DFS_NAMENODE_DU_RESERVED_KEY = "dfs.namenode.resource.du.reserved"; public static final String DFS_NAMENODE_DU_RESERVED_KEY = "dfs.namenode.resource.du.reserved";

View File

@ -288,7 +288,7 @@ public static BlockLocation[] locatedBlocks2Locations(LocatedBlocks blocks) {
* @return collection of nameservice Ids, or null if not specified * @return collection of nameservice Ids, or null if not specified
*/ */
public static Collection<String> getNameServiceIds(Configuration conf) { public static Collection<String> getNameServiceIds(Configuration conf) {
return conf.getTrimmedStringCollection(DFS_FEDERATION_NAMESERVICES); return conf.getTrimmedStringCollection(DFS_NAMESERVICES);
} }
/** /**
@ -873,7 +873,7 @@ public static String getSecondaryNameServiceId(Configuration conf) {
* Get the nameservice Id by matching the {@code addressKey} with the * Get the nameservice Id by matching the {@code addressKey} with the
* the address of the local node. * the address of the local node.
* *
* If {@link DFSConfigKeys#DFS_FEDERATION_NAMESERVICE_ID} is not specifically * If {@link DFSConfigKeys#DFS_NAMESERVICE_ID} is not specifically
* configured, and more than one nameservice Id is configured, this method * configured, and more than one nameservice Id is configured, this method
* determines the nameservice Id by matching the local node's address with the * determines the nameservice Id by matching the local node's address with the
* configured addresses. When a match is found, it returns the nameservice Id * configured addresses. When a match is found, it returns the nameservice Id
@ -885,7 +885,7 @@ public static String getSecondaryNameServiceId(Configuration conf) {
* @throws HadoopIllegalArgumentException on error * @throws HadoopIllegalArgumentException on error
*/ */
private static String getNameServiceId(Configuration conf, String addressKey) { private static String getNameServiceId(Configuration conf, String addressKey) {
String nameserviceId = conf.get(DFS_FEDERATION_NAMESERVICE_ID); String nameserviceId = conf.get(DFS_NAMESERVICE_ID);
if (nameserviceId != null) { if (nameserviceId != null) {
return nameserviceId; return nameserviceId;
} }
@ -957,7 +957,7 @@ static String[] getSuffixIDs(final Configuration conf, final String addressKey,
if (found > 1) { // Only one address must match the local address if (found > 1) { // Only one address must match the local address
String msg = "Configuration has multiple addresses that match " String msg = "Configuration has multiple addresses that match "
+ "local node's address. Please configure the system with " + "local node's address. Please configure the system with "
+ DFS_FEDERATION_NAMESERVICE_ID + " and " + DFS_NAMESERVICE_ID + " and "
+ DFS_HA_NAMENODE_ID_KEY; + DFS_HA_NAMENODE_ID_KEY;
throw new HadoopIllegalArgumentException(msg); throw new HadoopIllegalArgumentException(msg);
} }

View File

@ -142,7 +142,7 @@ public static Configuration getConfForOtherNode(
Preconditions.checkArgument(nsId != null, Preconditions.checkArgument(nsId != null,
"Could not determine namespace id. Please ensure that this " + "Could not determine namespace id. Please ensure that this " +
"machine is one of the machines listed as a NN RPC address, " + "machine is one of the machines listed as a NN RPC address, " +
"or configure " + DFSConfigKeys.DFS_FEDERATION_NAMESERVICE_ID); "or configure " + DFSConfigKeys.DFS_NAMESERVICE_ID);
Collection<String> nnIds = DFSUtil.getNameNodeIds(myConf, nsId); Collection<String> nnIds = DFSUtil.getNameNodeIds(myConf, nsId);
String myNNId = myConf.get(DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY); String myNNId = myConf.get(DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY);

View File

@ -63,7 +63,7 @@ public static void init() {
} }
private static void deprecate(String oldKey, String newKey) { private static void deprecate(String oldKey, String newKey) {
Configuration.addDeprecation(oldKey, new String[]{newKey}); Configuration.addDeprecation(oldKey, newKey);
} }
private static void addDeprecatedKeys() { private static void addDeprecatedKeys() {
@ -102,5 +102,7 @@ private static void addDeprecatedKeys() {
deprecate("dfs.block.size", DFSConfigKeys.DFS_BLOCK_SIZE_KEY); deprecate("dfs.block.size", DFSConfigKeys.DFS_BLOCK_SIZE_KEY);
deprecate("dfs.datanode.max.xcievers", DFSConfigKeys.DFS_DATANODE_MAX_RECEIVER_THREADS_KEY); deprecate("dfs.datanode.max.xcievers", DFSConfigKeys.DFS_DATANODE_MAX_RECEIVER_THREADS_KEY);
deprecate("io.bytes.per.checksum", DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY); deprecate("io.bytes.per.checksum", DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY);
deprecate("dfs.federation.nameservices", DFSConfigKeys.DFS_NAMESERVICES);
deprecate("dfs.federation.nameservice.id", DFSConfigKeys.DFS_NAMESERVICE_ID);
} }
} }

View File

@ -145,7 +145,7 @@ void joinAll() {
void refreshNamenodes(Configuration conf) void refreshNamenodes(Configuration conf)
throws IOException { throws IOException {
LOG.info("Refresh request received for nameservices: " LOG.info("Refresh request received for nameservices: "
+ conf.get(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES)); + conf.get(DFSConfigKeys.DFS_NAMESERVICES));
Map<String, Map<String, InetSocketAddress>> newAddressMap = Map<String, Map<String, InetSocketAddress>> newAddressMap =
DFSUtil.getNNServiceRpcAddresses(conf); DFSUtil.getNNServiceRpcAddresses(conf);

View File

@ -1138,7 +1138,7 @@ public static void initializeGenericKeys(Configuration conf,
if ((nameserviceId != null && !nameserviceId.isEmpty()) || if ((nameserviceId != null && !nameserviceId.isEmpty()) ||
(namenodeId != null && !namenodeId.isEmpty())) { (namenodeId != null && !namenodeId.isEmpty())) {
if (nameserviceId != null) { if (nameserviceId != null) {
conf.set(DFS_FEDERATION_NAMESERVICE_ID, nameserviceId); conf.set(DFS_NAMESERVICE_ID, nameserviceId);
} }
if (namenodeId != null) { if (namenodeId != null) {
conf.set(DFS_HA_NAMENODE_ID_KEY, namenodeId); conf.set(DFS_HA_NAMENODE_ID_KEY, namenodeId);

View File

@ -778,7 +778,7 @@
</property> </property>
<property> <property>
<name>dfs.federation.nameservices</name> <name>dfs.nameservices</name>
<value></value> <value></value>
<description> <description>
Comma-separated list of nameservices. Comma-separated list of nameservices.
@ -786,12 +786,12 @@
</property> </property>
<property> <property>
<name>dfs.federation.nameservice.id</name> <name>dfs.nameservice.id</name>
<value></value> <value></value>
<description> <description>
The ID of this nameservice. If the nameservice ID is not The ID of this nameservice. If the nameservice ID is not
configured or more than one nameservice is configured for configured or more than one nameservice is configured for
dfs.federation.nameservices it is determined automatically by dfs.nameservices it is determined automatically by
matching the local node's address with the configured address. matching the local node's address with the configured address.
</description> </description>
</property> </property>

View File

@ -707,7 +707,7 @@ public static void setFederatedConfiguration(MiniDFSCluster cluster,
info.nameserviceId), DFSUtil.createUri(HdfsConstants.HDFS_URI_SCHEME, info.nameserviceId), DFSUtil.createUri(HdfsConstants.HDFS_URI_SCHEME,
info.nameNode.getNameNodeAddress()).toString()); info.nameNode.getNameNodeAddress()).toString());
} }
conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, Joiner.on(",") conf.set(DFSConfigKeys.DFS_NAMESERVICES, Joiner.on(",")
.join(nameservices)); .join(nameservices));
} }

View File

@ -25,8 +25,8 @@
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_FEDERATION_NAMESERVICES; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICES;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_FEDERATION_NAMESERVICE_ID; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY;
@ -612,7 +612,7 @@ private void createNameNodesAndSetConf(MiniDFSNNTopology nnTopology,
} }
} }
if (!allNsIds.isEmpty()) { if (!allNsIds.isEmpty()) {
conf.set(DFS_FEDERATION_NAMESERVICES, Joiner.on(",").join(allNsIds)); conf.set(DFS_NAMESERVICES, Joiner.on(",").join(allNsIds));
} }
int nnCounter = 0; int nnCounter = 0;
@ -710,7 +710,7 @@ private void initNameNodeConf(Configuration conf,
boolean manageNameDfsDirs, int nnIndex) boolean manageNameDfsDirs, int nnIndex)
throws IOException { throws IOException {
if (nameserviceId != null) { if (nameserviceId != null) {
conf.set(DFS_FEDERATION_NAMESERVICE_ID, nameserviceId); conf.set(DFS_NAMESERVICE_ID, nameserviceId);
} }
if (nnId != null) { if (nnId != null) {
conf.set(DFS_HA_NAMENODE_ID_KEY, nnId); conf.set(DFS_HA_NAMENODE_ID_KEY, nnId);
@ -2111,9 +2111,9 @@ public NameNode addNameNode(Configuration conf, int namenodePort)
nameNodes = newlist; nameNodes = newlist;
String nameserviceId = NAMESERVICE_ID_PREFIX + (nnIndex + 1); String nameserviceId = NAMESERVICE_ID_PREFIX + (nnIndex + 1);
String nameserviceIds = conf.get(DFS_FEDERATION_NAMESERVICES); String nameserviceIds = conf.get(DFS_NAMESERVICES);
nameserviceIds += "," + nameserviceId; nameserviceIds += "," + nameserviceId;
conf.set(DFS_FEDERATION_NAMESERVICES, nameserviceIds); conf.set(DFS_NAMESERVICES, nameserviceIds);
String nnId = null; String nnId = null;
initNameNodeAddress(conf, nameserviceId, initNameNodeAddress(conf, nameserviceId,

View File

@ -100,7 +100,7 @@ public void testLocatedBlocks2Locations() {
private Configuration setupAddress(String key) { private Configuration setupAddress(String key) {
HdfsConfiguration conf = new HdfsConfiguration(); HdfsConfiguration conf = new HdfsConfiguration();
conf.set(DFS_FEDERATION_NAMESERVICES, "nn1"); conf.set(DFS_NAMESERVICES, "nn1");
conf.set(DFSUtil.addKeySuffixes(key, "nn1"), "localhost:9000"); conf.set(DFSUtil.addKeySuffixes(key, "nn1"), "localhost:9000");
return conf; return conf;
} }
@ -112,7 +112,7 @@ private Configuration setupAddress(String key) {
@Test @Test
public void getNameServiceId() { public void getNameServiceId() {
HdfsConfiguration conf = new HdfsConfiguration(); HdfsConfiguration conf = new HdfsConfiguration();
conf.set(DFS_FEDERATION_NAMESERVICE_ID, "nn1"); conf.set(DFS_NAMESERVICE_ID, "nn1");
assertEquals("nn1", DFSUtil.getNamenodeNameServiceId(conf)); assertEquals("nn1", DFSUtil.getNamenodeNameServiceId(conf));
} }
@ -157,7 +157,7 @@ public void getSecondaryNameServiceId() {
@Test(expected = HadoopIllegalArgumentException.class) @Test(expected = HadoopIllegalArgumentException.class)
public void testGetNameServiceIdException() { public void testGetNameServiceIdException() {
HdfsConfiguration conf = new HdfsConfiguration(); HdfsConfiguration conf = new HdfsConfiguration();
conf.set(DFS_FEDERATION_NAMESERVICES, "nn1,nn2"); conf.set(DFS_NAMESERVICES, "nn1,nn2");
conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "nn1"), conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "nn1"),
"localhost:9000"); "localhost:9000");
conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "nn2"), conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "nn2"),
@ -172,7 +172,7 @@ public void testGetNameServiceIdException() {
@Test @Test
public void testGetNameServiceIds() { public void testGetNameServiceIds() {
HdfsConfiguration conf = new HdfsConfiguration(); HdfsConfiguration conf = new HdfsConfiguration();
conf.set(DFS_FEDERATION_NAMESERVICES, "nn1,nn2"); conf.set(DFS_NAMESERVICES, "nn1,nn2");
Collection<String> nameserviceIds = DFSUtil.getNameServiceIds(conf); Collection<String> nameserviceIds = DFSUtil.getNameServiceIds(conf);
Iterator<String> it = nameserviceIds.iterator(); Iterator<String> it = nameserviceIds.iterator();
assertEquals(2, nameserviceIds.size()); assertEquals(2, nameserviceIds.size());
@ -183,11 +183,11 @@ public void testGetNameServiceIds() {
@Test @Test
public void testGetOnlyNameServiceIdOrNull() { public void testGetOnlyNameServiceIdOrNull() {
HdfsConfiguration conf = new HdfsConfiguration(); HdfsConfiguration conf = new HdfsConfiguration();
conf.set(DFS_FEDERATION_NAMESERVICES, "ns1,ns2"); conf.set(DFS_NAMESERVICES, "ns1,ns2");
assertNull(DFSUtil.getOnlyNameServiceIdOrNull(conf)); assertNull(DFSUtil.getOnlyNameServiceIdOrNull(conf));
conf.set(DFS_FEDERATION_NAMESERVICES, ""); conf.set(DFS_NAMESERVICES, "");
assertNull(DFSUtil.getOnlyNameServiceIdOrNull(conf)); assertNull(DFSUtil.getOnlyNameServiceIdOrNull(conf));
conf.set(DFS_FEDERATION_NAMESERVICES, "ns1"); conf.set(DFS_NAMESERVICES, "ns1");
assertEquals("ns1", DFSUtil.getOnlyNameServiceIdOrNull(conf)); assertEquals("ns1", DFSUtil.getOnlyNameServiceIdOrNull(conf));
} }
@ -199,7 +199,7 @@ public void testGetOnlyNameServiceIdOrNull() {
@Test @Test
public void testMultipleNamenodes() throws IOException { public void testMultipleNamenodes() throws IOException {
HdfsConfiguration conf = new HdfsConfiguration(); HdfsConfiguration conf = new HdfsConfiguration();
conf.set(DFS_FEDERATION_NAMESERVICES, "nn1,nn2"); conf.set(DFS_NAMESERVICES, "nn1,nn2");
// Test - configured list of namenodes are returned // Test - configured list of namenodes are returned
final String NN1_ADDRESS = "localhost:9000"; final String NN1_ADDRESS = "localhost:9000";
final String NN2_ADDRESS = "localhost:9001"; final String NN2_ADDRESS = "localhost:9001";
@ -270,8 +270,8 @@ public void testConfModificationFederationOnly() {
final HdfsConfiguration conf = new HdfsConfiguration(); final HdfsConfiguration conf = new HdfsConfiguration();
String nsId = "ns1"; String nsId = "ns1";
conf.set(DFS_FEDERATION_NAMESERVICES, nsId); conf.set(DFS_NAMESERVICES, nsId);
conf.set(DFS_FEDERATION_NAMESERVICE_ID, nsId); conf.set(DFS_NAMESERVICE_ID, nsId);
// Set the nameservice specific keys with nameserviceId in the config key // Set the nameservice specific keys with nameserviceId in the config key
for (String key : NameNode.NAMESERVICE_SPECIFIC_KEYS) { for (String key : NameNode.NAMESERVICE_SPECIFIC_KEYS) {
@ -299,8 +299,8 @@ public void testConfModificationFederationAndHa() {
String nsId = "ns1"; String nsId = "ns1";
String nnId = "nn1"; String nnId = "nn1";
conf.set(DFS_FEDERATION_NAMESERVICES, nsId); conf.set(DFS_NAMESERVICES, nsId);
conf.set(DFS_FEDERATION_NAMESERVICE_ID, nsId); conf.set(DFS_NAMESERVICE_ID, nsId);
conf.set(DFS_HA_NAMENODES_KEY_PREFIX + "." + nsId, nnId); conf.set(DFS_HA_NAMENODES_KEY_PREFIX + "." + nsId, nnId);
// Set the nameservice specific keys with nameserviceId in the config key // Set the nameservice specific keys with nameserviceId in the config key
@ -430,7 +430,7 @@ public void testHANameNodesWithFederation() throws URISyntaxException {
conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://ns1"); conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://ns1");
// Two nameservices, each with two NNs. // Two nameservices, each with two NNs.
conf.set(DFS_FEDERATION_NAMESERVICES, "ns1,ns2"); conf.set(DFS_NAMESERVICES, "ns1,ns2");
conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, "ns1"), conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, "ns1"),
"ns1-nn1,ns1-nn2"); "ns1-nn1,ns1-nn2");
conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, "ns2"), conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, "ns2"),
@ -491,7 +491,7 @@ public void getNameNodeServiceAddr() throws IOException {
final String NS1_NN2_HOST = "ns1-nn1.example.com:8020"; final String NS1_NN2_HOST = "ns1-nn1.example.com:8020";
final String NS1_NN2_HOST_SVC = "ns1-nn2.example.com:8021"; final String NS1_NN2_HOST_SVC = "ns1-nn2.example.com:8021";
conf.set(DFS_FEDERATION_NAMESERVICES, "ns1"); conf.set(DFS_NAMESERVICES, "ns1");
conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, "ns1"),"nn1,nn2"); conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, "ns1"),"nn1,nn2");
conf.set(DFSUtil.addKeySuffixes( conf.set(DFSUtil.addKeySuffixes(
@ -540,7 +540,7 @@ public void testGetNNUris() throws Exception {
final String NN1_SRVC_ADDR = "nn.example.com:8021"; final String NN1_SRVC_ADDR = "nn.example.com:8021";
final String NN2_ADDR = "nn2.example.com:8020"; final String NN2_ADDR = "nn2.example.com:8020";
conf.set(DFS_FEDERATION_NAMESERVICES, "ns1,ns2"); conf.set(DFS_NAMESERVICES, "ns1,ns2");
conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, "ns1"),"nn1,nn2"); conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, "ns1"),"nn1,nn2");
conf.set(DFSUtil.addKeySuffixes( conf.set(DFSUtil.addKeySuffixes(
DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn1"), NS1_NN1_ADDR); DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn1"), NS1_NN1_ADDR);

View File

@ -101,7 +101,7 @@ public void testSimpleSingleNS() throws Exception {
@Test @Test
public void testFederationRefresh() throws Exception { public void testFederationRefresh() throws Exception {
Configuration conf = new Configuration(); Configuration conf = new Configuration();
conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, conf.set(DFSConfigKeys.DFS_NAMESERVICES,
"ns1,ns2"); "ns1,ns2");
addNN(conf, "ns1", "mock1:8020"); addNN(conf, "ns1", "mock1:8020");
addNN(conf, "ns2", "mock1:8020"); addNN(conf, "ns2", "mock1:8020");
@ -112,7 +112,7 @@ public void testFederationRefresh() throws Exception {
log.setLength(0); log.setLength(0);
// Remove the first NS // Remove the first NS
conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, conf.set(DFSConfigKeys.DFS_NAMESERVICES,
"ns1"); "ns1");
bpm.refreshNamenodes(conf); bpm.refreshNamenodes(conf);
assertEquals( assertEquals(
@ -122,7 +122,7 @@ public void testFederationRefresh() throws Exception {
// Add back an NS -- this creates a new BPOS since the old // Add back an NS -- this creates a new BPOS since the old
// one for ns2 should have been previously retired // one for ns2 should have been previously retired
conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, conf.set(DFSConfigKeys.DFS_NAMESERVICES,
"ns1,ns2"); "ns1,ns2");
bpm.refreshNamenodes(conf); bpm.refreshNamenodes(conf);
assertEquals( assertEquals(

View File

@ -46,7 +46,7 @@ public void testDeleteBlockPool() throws Exception {
Configuration conf = new Configuration(); Configuration conf = new Configuration();
MiniDFSCluster cluster = null; MiniDFSCluster cluster = null;
try { try {
conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, conf.set(DFSConfigKeys.DFS_NAMESERVICES,
"namesServerId1,namesServerId2"); "namesServerId1,namesServerId2");
cluster = new MiniDFSCluster.Builder(conf) cluster = new MiniDFSCluster.Builder(conf)
.nnTopology(MiniDFSNNTopology.simpleFederatedTopology(2)) .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(2))
@ -79,7 +79,7 @@ public void testDeleteBlockPool() throws Exception {
} }
Configuration nn1Conf = cluster.getConfiguration(1); Configuration nn1Conf = cluster.getConfiguration(1);
nn1Conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, "namesServerId2"); nn1Conf.set(DFSConfigKeys.DFS_NAMESERVICES, "namesServerId2");
dn1.refreshNamenodes(nn1Conf); dn1.refreshNamenodes(nn1Conf);
assertEquals(1, dn1.getAllBpOs().length); assertEquals(1, dn1.getAllBpOs().length);
@ -155,7 +155,7 @@ public void testDfsAdminDeleteBlockPool() throws Exception {
Configuration conf = new Configuration(); Configuration conf = new Configuration();
MiniDFSCluster cluster = null; MiniDFSCluster cluster = null;
try { try {
conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, conf.set(DFSConfigKeys.DFS_NAMESERVICES,
"namesServerId1,namesServerId2"); "namesServerId1,namesServerId2");
cluster = new MiniDFSCluster.Builder(conf) cluster = new MiniDFSCluster.Builder(conf)
.nnTopology(MiniDFSNNTopology.simpleFederatedTopology(2)) .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(2))
@ -178,7 +178,7 @@ public void testDfsAdminDeleteBlockPool() throws Exception {
File dn1StorageDir2 = cluster.getInstanceStorageDir(0, 1); File dn1StorageDir2 = cluster.getInstanceStorageDir(0, 1);
Configuration nn1Conf = cluster.getConfiguration(0); Configuration nn1Conf = cluster.getConfiguration(0);
nn1Conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, "namesServerId1"); nn1Conf.set(DFSConfigKeys.DFS_NAMESERVICES, "namesServerId1");
dn1.refreshNamenodes(nn1Conf); dn1.refreshNamenodes(nn1Conf);
Assert.assertEquals(1, dn1.getAllBpOs().length); Assert.assertEquals(1, dn1.getAllBpOs().length);

View File

@ -105,7 +105,7 @@ public void testBlockScannerAfterRefresh() throws IOException,
namenodesBuilder.append(","); namenodesBuilder.append(",");
} }
conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, namenodesBuilder conf.set(DFSConfigKeys.DFS_NAMESERVICES, namenodesBuilder
.toString()); .toString());
DataNode dn = cluster.getDataNodes().get(0); DataNode dn = cluster.getDataNodes().get(0);
dn.refreshNamenodes(conf); dn.refreshNamenodes(conf);
@ -122,7 +122,7 @@ public void testBlockScannerAfterRefresh() throws IOException,
namenodesBuilder.append(DFSUtil.getNamenodeNameServiceId(cluster namenodesBuilder.append(DFSUtil.getNamenodeNameServiceId(cluster
.getConfiguration(2))); .getConfiguration(2)));
conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, namenodesBuilder conf.set(DFSConfigKeys.DFS_NAMESERVICES, namenodesBuilder
.toString()); .toString());
dn.refreshNamenodes(conf); dn.refreshNamenodes(conf);

View File

@ -1115,7 +1115,7 @@ public void testMultipleSecondaryNamenodes() throws IOException {
Configuration conf = new HdfsConfiguration(); Configuration conf = new HdfsConfiguration();
String nameserviceId1 = "ns1"; String nameserviceId1 = "ns1";
String nameserviceId2 = "ns2"; String nameserviceId2 = "ns2";
conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, nameserviceId1 conf.set(DFSConfigKeys.DFS_NAMESERVICES, nameserviceId1
+ "," + nameserviceId2); + "," + nameserviceId2);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.nnTopology(MiniDFSNNTopology.simpleFederatedTopology(2)) .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(2))

View File

@ -43,7 +43,7 @@ public void testIsValidRequestor() throws IOException {
KerberosName.setRules("RULE:[1:$1]\nRULE:[2:$1]"); KerberosName.setRules("RULE:[1:$1]\nRULE:[2:$1]");
// Set up generic HA configs. // Set up generic HA configs.
conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, "ns1"); conf.set(DFSConfigKeys.DFS_NAMESERVICES, "ns1");
conf.set(DFSUtil.addKeySuffixes(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX, conf.set(DFSUtil.addKeySuffixes(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX,
"ns1"), "nn1,nn2"); "ns1"), "nn1,nn2");

View File

@ -92,7 +92,7 @@ public void testGenericKeysForNameNodeFormat()
conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY,
"127.0.0.1:0"); "127.0.0.1:0");
conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, "ns1"); conf.set(DFSConfigKeys.DFS_NAMESERVICES, "ns1");
// Set a nameservice-specific configuration for name dir // Set a nameservice-specific configuration for name dir
File dir = new File(MiniDFSCluster.getBaseDirectory(), File dir = new File(MiniDFSCluster.getBaseDirectory(),

View File

@ -185,7 +185,7 @@ public static void setFailoverConfigurations(Configuration conf,
conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY,
logicalName, nameNodeId2), address2); logicalName, nameNodeId2), address2);
conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, logicalName); conf.set(DFSConfigKeys.DFS_NAMESERVICES, logicalName);
conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, logicalName), conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, logicalName),
nameNodeId1 + "," + nameNodeId2); nameNodeId1 + "," + nameNodeId2);
conf.set(DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX + "." + logicalName, conf.set(DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX + "." + logicalName,

View File

@ -57,7 +57,7 @@ public void testCheckpointerValidityChecks() throws Exception {
private Configuration getHAConf(String nsId, String host1, String host2) { private Configuration getHAConf(String nsId, String host1, String host2) {
Configuration conf = new Configuration(); Configuration conf = new Configuration();
conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, nsId); conf.set(DFSConfigKeys.DFS_NAMESERVICES, nsId);
conf.set(DFSUtil.addKeySuffixes( conf.set(DFSUtil.addKeySuffixes(
DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX, nsId), DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX, nsId),
"nn1,nn2"); "nn1,nn2");
@ -75,7 +75,7 @@ private Configuration getHAConf(String nsId, String host1, String host2) {
public void testGetOtherNNHttpAddress() { public void testGetOtherNNHttpAddress() {
// Use non-local addresses to avoid host address matching // Use non-local addresses to avoid host address matching
Configuration conf = getHAConf("ns1", "1.2.3.1", "1.2.3.2"); Configuration conf = getHAConf("ns1", "1.2.3.1", "1.2.3.2");
conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICE_ID, "ns1"); conf.set(DFSConfigKeys.DFS_NAMESERVICE_ID, "ns1");
// This is done by the NN before the StandbyCheckpointer is created // This is done by the NN before the StandbyCheckpointer is created
NameNode.initializeGenericKeys(conf, "ns1", "nn1"); NameNode.initializeGenericKeys(conf, "ns1", "nn1");

View File

@ -160,7 +160,7 @@ public void testDontOverWriteExistingDir() {
@Test @Test
public void testInitializeSharedEditsConfiguresGenericConfKeys() { public void testInitializeSharedEditsConfiguresGenericConfKeys() {
Configuration conf = new Configuration(); Configuration conf = new Configuration();
conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, "ns1"); conf.set(DFSConfigKeys.DFS_NAMESERVICES, "ns1");
conf.set(DFSUtil.addKeySuffixes(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX, conf.set(DFSUtil.addKeySuffixes(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX,
"ns1"), "nn1,nn2"); "ns1"), "nn1,nn2");
conf.set(DFSUtil.addKeySuffixes(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY, conf.set(DFSUtil.addKeySuffixes(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY,

View File

@ -64,8 +64,8 @@ public class TestDFSHAAdmin {
private HdfsConfiguration getHAConf() { private HdfsConfiguration getHAConf() {
HdfsConfiguration conf = new HdfsConfiguration(); HdfsConfiguration conf = new HdfsConfiguration();
conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, NSID); conf.set(DFSConfigKeys.DFS_NAMESERVICES, NSID);
conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICE_ID, NSID); conf.set(DFSConfigKeys.DFS_NAMESERVICE_ID, NSID);
conf.set(DFSUtil.addKeySuffixes( conf.set(DFSUtil.addKeySuffixes(
DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX, NSID), "nn1,nn2"); DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX, NSID), "nn1,nn2");
conf.set(DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY, "nn1"); conf.set(DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY, "nn1");

View File

@ -61,7 +61,7 @@ private void setupNameServices(HdfsConfiguration conf, int nameServiceIdCount) {
} }
nsList.append(getNameServiceId(i)); nsList.append(getNameServiceId(i));
} }
conf.set(DFS_FEDERATION_NAMESERVICES, nsList.toString()); conf.set(DFS_NAMESERVICES, nsList.toString());
} }
/** Set a given key with value as address, for all the nameServiceIds. /** Set a given key with value as address, for all the nameServiceIds.

View File

@ -131,7 +131,7 @@ HDFS Federation
** Configuration: ** Configuration:
<<Step 1>>: Add the following parameters to your configuration: <<Step 1>>: Add the following parameters to your configuration:
<<<dfs.federation.nameservices>>>: Configure with list of comma separated <<<dfs.nameservices>>>: Configure with list of comma separated
NameServiceIDs. This will be used by Datanodes to determine all the NameServiceIDs. This will be used by Datanodes to determine all the
Namenodes in the cluster. Namenodes in the cluster.
@ -164,7 +164,7 @@ HDFS Federation
---- ----
<configuration> <configuration>
<property> <property>
<name>dfs.federation.nameservices</name> <name>dfs.nameservices</name>
<value>ns1,ns2</value> <value>ns1,ns2</value>
</property> </property>
<property> <property>
@ -233,8 +233,7 @@ HDFS Federation
Follow the following steps: Follow the following steps:
* Add configuration parameter <<<dfs.federation.nameservices>>> to * Add configuration parameter <<<dfs.nameservices>>> to the configuration.
the configuration.
* Update the configuration with NameServiceID suffix. Configuration * Update the configuration with NameServiceID suffix. Configuration
key names have changed post release 0.20. You must use new configuration key names have changed post release 0.20. You must use new configuration

View File

@ -147,12 +147,12 @@ HDFS High Availability
<<hdfs-site.xml>> configuration file. <<hdfs-site.xml>> configuration file.
The order in which you set these configurations is unimportant, but the values The order in which you set these configurations is unimportant, but the values
you choose for <<dfs.federation.nameservices>> and you choose for <<dfs.nameservices>> and
<<dfs.ha.namenodes.[nameservice ID]>> will determine the keys of those that <<dfs.ha.namenodes.[nameservice ID]>> will determine the keys of those that
follow. Thus, you should decide on these values before setting the rest of the follow. Thus, you should decide on these values before setting the rest of the
configuration options. configuration options.
* <<dfs.federation.nameservices>> - the logical name for this new nameservice * <<dfs.nameservices>> - the logical name for this new nameservice
Choose a logical name for this nameservice, for example "mycluster", and use Choose a logical name for this nameservice, for example "mycluster", and use
this logical name for the value of this config option. The name you choose is this logical name for the value of this config option. The name you choose is
@ -165,7 +165,7 @@ HDFS High Availability
---- ----
<property> <property>
<name>dfs.federation.nameservices</name> <name>dfs.nameservices</name>
<value>mycluster</value> <value>mycluster</value>
</property> </property>
---- ----