diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java index c14d5d336e..62c5d8167a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java @@ -615,7 +615,7 @@ public static InetSocketAddress getNNAddress(String address) { public static InetSocketAddress getNNAddress(Configuration conf) { URI filesystemURI = FileSystem.getDefaultUri(conf); - return getNNAddress(filesystemURI); + return getNNAddressCheckLogical(conf, filesystemURI); } /** @@ -638,6 +638,26 @@ public static InetSocketAddress getNNAddress(URI filesystemURI) { return getNNAddress(authority); } + /** + * Get the NN address from the URI. If the uri is logical, default address is + * returned. Otherwise return the DNS-resolved address of the URI. + * + * @param conf configuration + * @param filesystemURI URI of the file system + * @return address of file system + */ + public static InetSocketAddress getNNAddressCheckLogical(Configuration conf, + URI filesystemURI) { + InetSocketAddress retAddr; + if (HAUtilClient.isLogicalUri(conf, filesystemURI)) { + retAddr = InetSocketAddress.createUnresolved(filesystemURI.getAuthority(), + HdfsClientConfigKeys.DFS_NAMENODE_RPC_PORT_DEFAULT); + } else { + retAddr = getNNAddress(filesystemURI); + } + return retAddr; + } + public static URI getNNUri(InetSocketAddress namenode) { int port = namenode.getPort(); String portString = diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java index 39b188fbbe..5ca7030acd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/NameNodeProxiesClient.java @@ -320,7 +320,7 @@ public static ProxyAndInfo createHAProxy( DFSUtilClient.getNNAddress(nameNodeUri)); } return new ProxyAndInfo<>(proxy, dtService, - DFSUtilClient.getNNAddress(nameNodeUri)); + DFSUtilClient.getNNAddressCheckLogical(conf, nameNodeUri)); } public static ClientProtocol createNonHAProxyWithClientProtocol( diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 8625ecf28d..336db0328c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -2285,6 +2285,9 @@ Release 2.8.0 - UNRELEASED HDFS-9401. Fix findbugs warnings in BlockRecoveryWorker. (Brahma Reddy Battula via waltersu4549) + HDFS-9364. Unnecessary DNS resolution attempts when creating NameNodeProxies. + (Xiao Chen via zhz) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java index b098711107..60ffe30a74 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java @@ -46,6 +46,7 @@ import org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider; import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; import org.apache.hadoop.hdfs.server.namenode.ha.IPFailoverProxyProvider; +import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.retry.FailoverProxyProvider; import org.apache.hadoop.net.ConnectTimeoutException; @@ -291,6 +292,40 @@ public void testFileContextDoesntDnsResolveLogicalURI() throws Exception { Mockito.verify(spyNS, Mockito.never()).lookupAllHostAddr(Mockito.eq(logicalHost)); } + /** + * Test that creating proxy doesn't ever try to DNS-resolve the logical URI. + * Regression test for HDFS-9364. + */ + @Test(timeout=60000) + public void testCreateProxyDoesntDnsResolveLogicalURI() throws IOException { + final NameService spyNS = spyOnNameService(); + final Configuration conf = new HdfsConfiguration(); + final String service = "nameservice1"; + final String namenode = "namenode113"; + conf.set(DFSConfigKeys.DFS_NAMESERVICES, service); + conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "hdfs://" + service); + conf.set( + HdfsClientConfigKeys.Failover.PROXY_PROVIDER_KEY_PREFIX + "." + service, + "org.apache.hadoop.hdfs.server.namenode.ha." + + "ConfiguredFailoverProxyProvider"); + conf.set(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + "." + service, + namenode); + conf.set(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + "." + service + "." + + namenode, "localhost:8020"); + + // call createProxy implicitly and explicitly + Path p = new Path("/"); + p.getFileSystem(conf); + NameNodeProxiesClient.createProxyWithClientProtocol(conf, + FileSystem.getDefaultUri(conf), null); + NameNodeProxies.createProxy(conf, FileSystem.getDefaultUri(conf), + NamenodeProtocol.class, null); + + // Ensure that the logical hostname was never resolved. + Mockito.verify(spyNS, Mockito.never()).lookupAllHostAddr( + Mockito.eq(service)); + } + /** Dummy implementation of plain FailoverProxyProvider */ public static class DummyLegacyFailoverProxyProvider implements FailoverProxyProvider {