big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-2819. Document new HA-related configs in hdfs-default.xml. Contributed by Eli Collins

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-1623@1240914 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Eli Collins 2012-02-06 08:21:06 +00:00
parent a660c5dd2c
commit c17b4f8eef
13 changed files with 167 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/NodeFencer.java
View File

@ -55,7 +55,7 @@
@InterfaceStability.Evolving
public class NodeFencer {
public static final String CONF_METHODS_KEY =
"dfs.namenode.ha.fencing.methods";
"dfs.ha.fencing.methods";
private static final String CLASS_RE = "([a-zA-Z0-9\\.\\$]+)";
private static final Pattern CLASS_WITH_ARGUMENT =
@ -92,7 +92,7 @@ public static NodeFencer create(Configuration conf)
}
public boolean fence(InetSocketAddress serviceAddr) {
LOG.info("====== Beginning NameNode Fencing Process... ======");
LOG.info("====== Beginning Service Fencing Process... ======");
int i = 0;
for (FenceMethodWithArg method : methods) {
LOG.info("Trying method " + (++i) + "/" + methods.size() +": " + method);
@ -112,7 +112,7 @@ public boolean fence(InetSocketAddress serviceAddr) {
LOG.warn("Fencing method " + method + " was unsuccessful.");
}
LOG.error("Unable to fence NameNode by any configured method.");
LOG.error("Unable to fence service by any configured method.");
return false;
}

6
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/SshFenceByTcpPort.java
View File

@ -52,7 +52,7 @@
* with ssh.
* <p>
* In order to achieve passwordless SSH, the operator must also configure
* <code>dfs.namenode.ha.fencing.ssh.private-key-files<code> to point to an
* <code>dfs.ha.fencing.ssh.private-key-files<code> to point to an
* SSH key that has passphrase-less access to the given username and host.
*/
public class SshFenceByTcpPort extends Configured
@ -62,11 +62,11 @@ public class SshFenceByTcpPort extends Configured
SshFenceByTcpPort.class);
static final String CONF_CONNECT_TIMEOUT_KEY =
"dfs.namenode.ha.fencing.ssh.connect-timeout";
"dfs.ha.fencing.ssh.connect-timeout";
private static final int CONF_CONNECT_TIMEOUT_DEFAULT =
30*1000;
static final String CONF_IDENTITIES_KEY =
"dfs.namenode.ha.fencing.ssh.private-key-files";
"dfs.ha.fencing.ssh.private-key-files";
/**
* Verify that the argument, if given, in the conf is parseable.

26
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
View File

@ -842,4 +842,30 @@
</description>
</property>
<property>
<name>dfs.ha.fencing.methods</name>
<value></value>
<description>
List of fencing methods to use for service fencing. May contain
builtin methods (eg shell and sshfence) or user-defined method.
</description>
</property>
<property>
<name>dfs.ha.fencing.ssh.connect-timeout</name>
<value>30000</value>
<description>
SSH connection timeout, in milliseconds, to use with the builtin
sshfence fencer.
</description>
</property>
<property>
<name>dfs.ha.fencing.ssh.private-key-files</name>
<value></value>
<description>
The SSH private key files to use with the builtin sshfence fencer.
</description>
</property>
</configuration>

2
hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt
View File

@ -168,3 +168,5 @@ HDFS-2890. DFSUtil#getSuffixIDs should skip unset configurations. (atm)
HDFS-2792. Make fsck work. (atm)
HDFS-2808. HA: haadmin should use namenode ids. (eli)
HDFS-2819. Document new HA-related configs in hdfs-default.xml. (eli)

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
View File

@ -48,6 +48,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_CLIENT_WRITE_REPLACE_DATANODE_ON_FAILURE_POLICY_DEFAULT = "DEFAULT";
public static final String DFS_CLIENT_SOCKET_CACHE_CAPACITY_KEY = "dfs.client.socketcache.capacity";
public static final int DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT = 16;
// HA related configuration
public static final String DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX = "dfs.client.failover.proxy.provider";
public static final String DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY = "dfs.client.failover.max.attempts";
public static final int DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT = 15;
@ -329,7 +331,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY = "dfs.block.local-path-access.user";
// HA related configuration
public static final String DFS_HA_NAMENODES_KEY = "dfs.ha.namenodes";
public static final String DFS_HA_NAMENODES_KEY_PREFIX = "dfs.ha.namenodes";
public static final String DFS_HA_NAMENODE_ID_KEY = "dfs.ha.namenode.id";
public static final String DFS_HA_STANDBY_CHECKPOINTS_KEY = "dfs.ha.standby.checkpoints";
public static final boolean DFS_HA_STANDBY_CHECKPOINTS_DEFAULT = true;

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
View File

@ -349,7 +349,7 @@ private static Collection<String> emptyAsSingletonNull(Collection<String> coll)
* @return collection of namenode Ids
*/
public static Collection<String> getNameNodeIds(Configuration conf, String nsId) {
String key = addSuffix(DFS_HA_NAMENODES_KEY, nsId);
String key = addSuffix(DFS_HA_NAMENODES_KEY_PREFIX, nsId);
return conf.getTrimmedStringCollection(key);
}

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HAUtil.java
View File

@ -139,7 +139,7 @@ public static Configuration getConfForOtherNode(
Preconditions.checkArgument(nnIds != null,
"Could not determine namenode ids in namespace '%s'. " +
"Please configure " +
DFSUtil.addKeySuffixes(DFSConfigKeys.DFS_HA_NAMENODES_KEY,
DFSUtil.addKeySuffixes(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX,
nsId),
nsId);
Preconditions.checkArgument(nnIds.size() == 2,

122
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
View File

@ -33,9 +33,11 @@
<property>
<name>dfs.namenode.logging.level</name>
<value>info</value>
<description>The logging level for dfs namenode. Other values are "dir"(trac
e namespace mutations), "block"(trace block under/over replications and block
creations/deletions), or "all".</description>
<description>
The logging level for dfs namenode. Other values are "dir" (trace
namespace mutations), "block" (trace block under/over replications
and block creations/deletions), or "all".
</description>
</property>
<property>
@ -710,4 +712,118 @@ creations/deletions), or "all".</description>
</description>
</property>
<property>
<name>dfs.client.failover.max.attempts</name>
<value>15</value>
<description>
Expert only. The number of client failover attempts that should be
made before the failover is considered failed.
</description>
</property>
<property>
<name>dfs.client.failover.sleep.base.millis</name>
<value>500</value>
<description>
Expert only. The time to wait, in milliseconds, between failover
attempts increases exponentially as a function of the number of
attempts made so far, with a random factor of +/- 50%. This option
specifies the base value used in the failover calculation. The
first failover will retry immediately. The 2nd failover attempt
will delay at least dfs.client.failover.sleep.base.millis
milliseconds. And so on.
</description>
</property>
<property>
<name>dfs.client.failover.sleep.max.millis</name>
<value>15000</value>
<description>
Expert only. The time to wait, in milliseconds, between failover
attempts increases exponentially as a function of the number of
attempts made so far, with a random factor of +/- 50%. This option
specifies the maximum value to wait between failovers.
Specifically, the time between two failover attempts will not
exceed +/- 50% of dfs.client.failover.sleep.max.millis
milliseconds.
</description>
</property>
<property>
<name>dfs.client.failover.connection.retries</name>
<value>0</value>
<description>
Expert only. Indicates the number of retries a failover IPC client
will make to establish a server connection.
</description>
</property>
<property>
<name>dfs.client.failover.connection.retries.on.timeouts</name>
<value>0</value>
<description>
Expert only. The number of retry attempts a failover IPC client
will make on socket timeout when establishing a server connection.
</description>
</property>
<property>
<name>dfs.federation.nameservices</name>
<value></value>
<description>
Comma-separated list of nameservices.
</description>
</property>
<property>
<name>dfs.federation.nameservice.id</name>
<value></value>
<description>
The ID of this nameservice. If the nameservice ID is not
configured or more than one nameservice is configured for
dfs.federation.nameservices it is determined automatically by
matching the local node's address with the configured address.
</description>
</property>
<property>
<name>dfs.ha.namenodes.EXAMPLENAMESERVICE</name>
<value></value>
<description>
The prefix for a given nameservice, contains a comma-separated
list of namenodes for a given nameservice (eg EXAMPLENAMESERVICE).
</description>
</property>
<property>
<name>dfs.ha.namenode.id</name>
<value></value>
<description>
The ID of this namenode. If the namenode ID is not configured it
is determined automatically by matching the local node's address
with the configured address.
</description>
</property>
<property>
<name>dfs.ha.log-roll.period</name>
<value>120</value>
<description>
How often, in seconds, the StandbyNode should ask the active to
roll edit logs. Since the StandbyNode only reads from finalized
log segments, the StandbyNode will only be as up-to-date as how
often the logs are rolled. Note that failover triggers a log roll
so the StandbyNode will be up to date before it becomes active.
</description>
</property>
<property>
<name>dfs.ha.tail-edits.period</name>
<value>60</value>
<description>
How often, in seconds, the StandbyNode should check for new
finalized log segments in the shared edits log.
</description>
</property>
</configuration>

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
View File

@ -618,7 +618,7 @@ private void createNameNodesAndSetConf(MiniDFSNNTopology nnTopology,
// If HA is enabled on this nameservice, enumerate all the namenodes
// in the configuration. Also need to set a shared edits dir
if (nnIds.size() > 1) {
conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY, nameservice.getId()),
conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, nameservice.getId()),
Joiner.on(",").join(nnIds));
if (manageNameDfsDirs) {
URI sharedEditsUri = getSharedEditsDir(nnCounter, nnCounter+nnIds.size()-1);

8
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
View File

@ -303,7 +303,7 @@ public void testConfModificationFederationAndHa() {
conf.set(DFS_FEDERATION_NAMESERVICES, nsId);
conf.set(DFS_FEDERATION_NAMESERVICE_ID, nsId);
conf.set(DFS_HA_NAMENODES_KEY + "." + nsId, nnId);
conf.set(DFS_HA_NAMENODES_KEY_PREFIX + "." + nsId, nnId);
// Set the nameservice specific keys with nameserviceId in the config key
for (String key : NameNode.NAMESERVICE_SPECIFIC_KEYS) {
@ -380,9 +380,9 @@ public void testHANameNodesWithFederation() {
// Two nameservices, each with two NNs.
conf.set(DFS_FEDERATION_NAMESERVICES, "ns1,ns2");
conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY, "ns1"),
conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, "ns1"),
"ns1-nn1,ns1-nn2");
conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY, "ns2"),
conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, "ns2"),
"ns2-nn1,ns2-nn2");
conf.set(DFSUtil.addKeySuffixes(
DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "ns1-nn1"),
@ -430,7 +430,7 @@ public void getNameNodeServiceAddr() throws IOException {
final String NS1_NN2_HOST_SVC = "ns1-nn2.example.com:8021";
conf.set(DFS_FEDERATION_NAMESERVICES, "ns1");
conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY, "ns1"),"nn1,nn2");
conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, "ns1"),"nn1,nn2");
conf.set(DFSUtil.addKeySuffixes(
DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn1"), NS1_NN1_HOST);

4
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/HATestUtil.java
View File

@ -18,7 +18,7 @@
package org.apache.hadoop.hdfs.server.namenode.ha;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY;
import java.io.IOException;
@ -171,7 +171,7 @@ public static void setFailoverConfigurations(MiniDFSCluster cluster,
logicalName, nameNodeId2), address2);
conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, logicalName);
conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY, logicalName),
conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, logicalName),
nameNodeId1 + "," + nameNodeId2);
conf.set(DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX + "." + logicalName,
ConfiguredFailoverProxyProvider.class.getName());

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAConfiguration.java
View File

@ -61,7 +61,7 @@ public void testGetOtherNNHttpAddress() {
conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, NSID);
conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICE_ID, NSID);
conf.set(DFSUtil.addKeySuffixes(
DFSConfigKeys.DFS_HA_NAMENODES_KEY, NSID),
DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX, NSID),
"nn1,nn2");
conf.set(DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY, "nn1");
conf.set(DFSUtil.addKeySuffixes(

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java
View File

@ -59,7 +59,7 @@ private HdfsConfiguration getHAConf() {
conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, NSID);
conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICE_ID, NSID);
conf.set(DFSUtil.addKeySuffixes(
DFSConfigKeys.DFS_HA_NAMENODES_KEY, NSID), "nn1,nn2");
DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX, NSID), "nn1,nn2");
conf.set(DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY, "nn1");
conf.set(DFSUtil.addKeySuffixes(
DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY, NSID, "nn1"),