big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-9364. Unnecessary DNS resolution attempts when creating NameNodeProxies. Contributed by Xiao Chen.

Browse Source 
Change-Id: I9e42f724f27924cf73891425a832de37ce014a1e
This commit is contained in:
Zhe Zhang 2015-11-10 09:55:29 -08:00
parent 0eb9c60c5b
commit 73b94d7899
4 changed files with 60 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
View File

@ -615,7 +615,7 @@ public static InetSocketAddress getNNAddress(String address) {
public static InetSocketAddress getNNAddress(Configuration conf) {
URI filesystemURI = FileSystem.getDefaultUri(conf);
return getNNAddress(filesystemURI);
return getNNAddressCheckLogical(conf, filesystemURI);
}
/**
@ -638,6 +638,26 @@ public static InetSocketAddress getNNAddress(URI filesystemURI) {
return getNNAddress(authority);
}
/**
* Get the NN address from the URI. If the uri is logical, default address is
* returned. Otherwise return the DNS-resolved address of the URI.
*
* @param conf configuration
* @param filesystemURI URI of the file system
* @return address of file system
*/
public static InetSocketAddress getNNAddressCheckLogical(Configuration conf,
URI filesystemURI) {
InetSocketAddress retAddr;
if (HAUtilClient.isLogicalUri(conf, filesystemURI)) {
retAddr = InetSocketAddress.createUnresolved(filesystemURI.getAuthority(),
HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT);
} else {
retAddr = getNNAddress(filesystemURI);
}
return retAddr;
}
public static URI getNNUri(InetSocketAddress namenode) {
int port = namenode.getPort();
String portString =

2
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java
View File

@ -320,7 +320,7 @@ public static <T> ProxyAndInfo<T> createHAProxy(
DFSUtilClient.getNNAddress(nameNodeUri));
}
return new ProxyAndInfo<>(proxy, dtService,
DFSUtilClient.getNNAddress(nameNodeUri));
DFSUtilClient.getNNAddressCheckLogical(conf, nameNodeUri));
}
public static ClientProtocol createNonHAProxyWithClientProtocol(

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -2285,6 +2285,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9401. Fix findbugs warnings in BlockRecoveryWorker.
(Brahma Reddy Battula via waltersu4549)
HDFS-9364. Unnecessary DNS resolution attempts when creating NameNodeProxies.
(Xiao Chen via zhz)
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES

35
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java
View File

@ -46,6 +46,7 @@
import org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider;
import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
import org.apache.hadoop.hdfs.server.namenode.ha.IPFailoverProxyProvider;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.retry.FailoverProxyProvider;
import org.apache.hadoop.net.ConnectTimeoutException;
@ -291,6 +292,40 @@ public void testFileContextDoesntDnsResolveLogicalURI() throws Exception {
Mockito.verify(spyNS, Mockito.never()).lookupAllHostAddr(Mockito.eq(logicalHost));
}
/**
* Test that creating proxy doesn't ever try to DNS-resolve the logical URI.
* Regression test for HDFS-9364.
*/
@Test(timeout=60000)
public void testCreateProxyDoesntDnsResolveLogicalURI() throws IOException {
final NameService spyNS = spyOnNameService();
final Configuration conf = new HdfsConfiguration();
final String service = "nameservice1";
final String namenode = "namenode113";
conf.set(DFSConfigKeys.DFS_NAMESERVICES, service);
conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "hdfs://" + service);
conf.set(
HdfsClientConfigKeys.Failover.PROXY_PROVIDER_KEY_PREFIX + "." + service,
"org.apache.hadoop.hdfs.server.namenode.ha."
+ "ConfiguredFailoverProxyProvider");
conf.set(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + "." + service,
namenode);
conf.set(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + "." + service + "."
+ namenode, "localhost:8020");
// call createProxy implicitly and explicitly
Path p = new Path("/");
p.getFileSystem(conf);
NameNodeProxiesClient.createProxyWithClientProtocol(conf,
FileSystem.getDefaultUri(conf), null);
NameNodeProxies.createProxy(conf, FileSystem.getDefaultUri(conf),
NamenodeProtocol.class, null);
// Ensure that the logical hostname was never resolved.
Mockito.verify(spyNS, Mockito.never()).lookupAllHostAddr(
Mockito.eq(service));
}
/** Dummy implementation of plain FailoverProxyProvider */
public static class DummyLegacyFailoverProxyProvider<T>
implements FailoverProxyProvider<T> {