HDFS-5338. Add a conf to disable hostname check in datanode registration.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1532468 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Tsz-wo Sze 2013-10-15 18:21:22 +00:00
parent 4407d5f65f
commit 0a5f0efcb4
5 changed files with 44 additions and 8 deletions

View File

@ -313,6 +313,9 @@ Release 2.3.0 - UNRELEASED
HDFS-5323. Remove some deadcode in BlockManager (Colin Patrick McCabe) HDFS-5323. Remove some deadcode in BlockManager (Colin Patrick McCabe)
HDFS-5338. Add a conf to disable hostname check in datanode registration.
(szetszwo)
OPTIMIZATIONS OPTIMIZATIONS
HDFS-5239. Allow FSNamesystem lock fairness to be configurable (daryn) HDFS-5239. Allow FSNamesystem lock fairness to be configurable (daryn)

View File

@ -194,6 +194,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_KEY = "dfs.datanode.socket.reuse.keepalive"; public static final String DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_KEY = "dfs.datanode.socket.reuse.keepalive";
public static final int DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_DEFAULT = 1000; public static final int DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_DEFAULT = 1000;
public static final String DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_KEY = "dfs.namenode.datanode.registration.ip-hostname-check";
public static final boolean DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_DEFAULT = true;
// Whether to enable datanode's stale state detection and usage for reads // Whether to enable datanode's stale state detection and usage for reads
public static final String DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_KEY = "dfs.namenode.avoid.read.stale.datanode"; public static final String DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_KEY = "dfs.namenode.avoid.read.stale.datanode";
public static final boolean DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_DEFAULT = false; public static final boolean DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_DEFAULT = false;

View File

@ -142,6 +142,7 @@ public class DatanodeManager {
*/ */
private boolean hasClusterEverBeenMultiRack = false; private boolean hasClusterEverBeenMultiRack = false;
private final boolean checkIpHostnameInRegistration;
/** /**
* The number of datanodes for each software version. This list should change * The number of datanodes for each software version. This list should change
* during rolling upgrades. * during rolling upgrades.
@ -210,6 +211,12 @@ public class DatanodeManager {
LOG.info(DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_KEY LOG.info(DFSConfigKeys.DFS_BLOCK_INVALIDATE_LIMIT_KEY
+ "=" + this.blockInvalidateLimit); + "=" + this.blockInvalidateLimit);
this.checkIpHostnameInRegistration = conf.getBoolean(
DFSConfigKeys.DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_KEY,
DFSConfigKeys.DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_DEFAULT);
LOG.info(DFSConfigKeys.DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_KEY
+ "=" + checkIpHostnameInRegistration);
this.avoidStaleDataNodesForRead = conf.getBoolean( this.avoidStaleDataNodesForRead = conf.getBoolean(
DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_KEY, DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_KEY,
DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_DEFAULT); DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_DEFAULT);
@ -733,11 +740,13 @@ public void registerDatanode(DatanodeRegistration nodeReg)
// Mostly called inside an RPC, update ip and peer hostname // Mostly called inside an RPC, update ip and peer hostname
String hostname = dnAddress.getHostName(); String hostname = dnAddress.getHostName();
String ip = dnAddress.getHostAddress(); String ip = dnAddress.getHostAddress();
if (!isNameResolved(dnAddress)) { if (checkIpHostnameInRegistration && !isNameResolved(dnAddress)) {
// Reject registration of unresolved datanode to prevent performance // Reject registration of unresolved datanode to prevent performance
// impact of repetitive DNS lookups later. // impact of repetitive DNS lookups later.
LOG.warn("Unresolved datanode registration from " + ip); final String message = "hostname cannot be resolved (ip="
throw new DisallowedDatanodeException(nodeReg); + ip + ", hostname=" + hostname + ")";
LOG.warn("Unresolved datanode registration: " + message);
throw new DisallowedDatanodeException(nodeReg, message);
} }
// update node registration with the ip and hostname from rpc request // update node registration with the ip and hostname from rpc request
nodeReg.setIpAddr(ip); nodeReg.setIpAddr(ip);
@ -1185,17 +1194,17 @@ public List<DatanodeDescriptor> getDatanodeListForReport(
/** /**
* Checks if name resolution was successful for the given address. If IP * Checks if name resolution was successful for the given address. If IP
* address and host name are the same, then it means name resolution has * address and host name are the same, then it means name resolution has
* failed. As a special case, the loopback address is also considered * failed. As a special case, local addresses are also considered
* acceptable. This is particularly important on Windows, where 127.0.0.1 does * acceptable. This is particularly important on Windows, where 127.0.0.1 does
* not resolve to "localhost". * not resolve to "localhost".
* *
* @param address InetAddress to check * @param address InetAddress to check
* @return boolean true if name resolution successful or address is loopback * @return boolean true if name resolution successful or address is local
*/ */
private static boolean isNameResolved(InetAddress address) { private static boolean isNameResolved(InetAddress address) {
String hostname = address.getHostName(); String hostname = address.getHostName();
String ip = address.getHostAddress(); String ip = address.getHostAddress();
return !hostname.equals(ip) || address.isLoopbackAddress(); return !hostname.equals(ip) || NetUtils.isLocalAddress(address);
} }
private void setDatanodeDead(DatanodeDescriptor node) { private void setDatanodeDead(DatanodeDescriptor node) {

View File

@ -37,7 +37,12 @@ public class DisallowedDatanodeException extends IOException {
/** for java.io.Serializable */ /** for java.io.Serializable */
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
public DisallowedDatanodeException(DatanodeID nodeID, String reason) {
super("Datanode denied communication with namenode because "
+ reason + ": " + nodeID);
}
public DisallowedDatanodeException(DatanodeID nodeID) { public DisallowedDatanodeException(DatanodeID nodeID) {
super("Datanode denied communication with namenode: " + nodeID); this(nodeID, "the host is not in the include-list");
} }
} }

View File

@ -569,6 +569,22 @@
</description> </description>
</property> </property>
<property>
<name>dfs.namenode.datanode.registration.ip-hostname-check</name>
<value>true</value>
<description>
If true (the default), then the namenode requires that a connecting
datanode's address must be resolved to a hostname. If necessary, a reverse
DNS lookup is performed. All attempts to register a datanode from an
unresolvable address are rejected.
It is recommended that this setting be left on to prevent accidental
registration of datanodes listed by hostname in the excludes file during a
DNS outage. Only set this to false in environments where there is no
infrastructure to support reverse DNS lookup.
</description>
</property>
<property> <property>
<name>dfs.namenode.decommission.interval</name> <name>dfs.namenode.decommission.interval</name>
<value>30</value> <value>30</value>