From 7b5b783f66f32012c00bef7593851392dd8cf2d5 Mon Sep 17 00:00:00 2001 From: Inigo Goiri Date: Wed, 3 Apr 2019 16:11:13 -0700 Subject: [PATCH] HDFS-14327. Using FQDN instead of IP to access servers with DNS resolving. Contributed by Fengnan Li. --- .../hadoop/net/DNSDomainNameResolver.java | 33 ++++++++++++- .../apache/hadoop/net/DomainNameResolver.java | 23 +++++++++ .../hadoop/net/MockDomainNameResolver.java | 36 ++++++++++++-- .../hdfs/client/HdfsClientConfigKeys.java | 2 + .../ha/AbstractNNFailoverProxyProvider.java | 18 ++++--- .../TestConfiguredFailoverProxyProvider.java | 47 ++++++++++++------- .../src/main/resources/hdfs-default.xml | 14 ++++++ 7 files changed, 145 insertions(+), 28 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSDomainNameResolver.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSDomainNameResolver.java index bb1aa90340..5866e2960f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSDomainNameResolver.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNSDomainNameResolver.java @@ -22,8 +22,9 @@ import java.net.UnknownHostException; /** - * DNSDomainNameResolver takes one domain name and returns all of the IP - * addresses from the underlying DNS service. + * DNSDomainNameResolver wraps up the default DNS service for forward/reverse + * DNS lookup. It also provides a function to resolve a host name to all of + * fully qualified domain names belonging to the IPs from this host name */ public class DNSDomainNameResolver implements DomainNameResolver { @Override @@ -31,4 +32,32 @@ public InetAddress[] getAllByDomainName(String domainName) throws UnknownHostException { return InetAddress.getAllByName(domainName); } + + @Override + public String getHostnameByIP(InetAddress address) { + String host = address.getCanonicalHostName(); + if (host != null && host.length() != 0 + && host.charAt(host.length()-1) == '.') { + host = host.substring(0, host.length()-1); + } + return host; + } + + @Override + public String[] getAllResolvedHostnameByDomainName( + String domainName, boolean useFQDN) throws UnknownHostException { + InetAddress[] addresses = getAllByDomainName(domainName); + String[] hosts = new String[addresses.length]; + if (useFQDN) { + for (int i = 0; i < addresses.length; i++) { + hosts[i] = getHostnameByIP(addresses[i]); + } + } else { + for (int i = 0; i < addresses.length; i++) { + hosts[i] = addresses[i].getHostAddress(); + } + } + + return hosts; + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DomainNameResolver.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DomainNameResolver.java index 6d2d800f9c..4c44e9da4c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DomainNameResolver.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DomainNameResolver.java @@ -36,4 +36,27 @@ public interface DomainNameResolver { */ InetAddress[] getAllByDomainName(String domainName) throws UnknownHostException; + + /** + * Reverse lookup an IP address and get the fully qualified domain name(fqdn). + * + * @param address + * @return fully qualified domain name + */ + String getHostnameByIP(InetAddress address); + + /** + * This function combines getAllByDomainName and getHostnameByIP, for a single + * domain name, it will first do a forward lookup to get all of IP addresses, + * then for each IP address, it will do a reverse lookup to get the fqdn. + * This function is necessary in secure environment since Kerberos uses fqdn + * in the service principal instead of IP. + * + * @param domainName + * @return all fully qualified domain names belonging to the IPs resolved from + * the input domainName + * @throws UnknownHostException + */ + String[] getAllResolvedHostnameByDomainName( + String domainName, boolean useFQDN) throws UnknownHostException; } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/MockDomainNameResolver.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/MockDomainNameResolver.java index cb55ae082b..aa93709337 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/MockDomainNameResolver.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/MockDomainNameResolver.java @@ -19,14 +19,16 @@ import java.net.InetAddress; import java.net.UnknownHostException; +import java.util.HashMap; import java.util.Map; import java.util.TreeMap; import com.google.common.annotations.VisibleForTesting; /** - * This mock resolver class returns the predefined resolving results. - * By default it uses a default "test.foo.bar" domain with two IP addresses. + * This mock resolver class returns the predefined resolving/reverse lookup + * results. By default it uses a default "test.foo.bar" domain with two + * IP addresses. */ public class MockDomainNameResolver implements DomainNameResolver { @@ -37,16 +39,21 @@ public class MockDomainNameResolver implements DomainNameResolver { public static final byte[] BYTE_ADDR_2 = new byte[]{10, 1, 1, 2}; public static final String ADDR_1 = "10.1.1.1"; public static final String ADDR_2 = "10.1.1.2"; + public static final String FQDN_1 = "host01.com"; + public static final String FQDN_2 = "host02.com"; /** Internal mapping of domain names and IP addresses. */ private Map addrs = new TreeMap<>(); - + /** Internal mapping from IP addresses to fqdns. */ + private Map ptrMap = new HashMap<>(); public MockDomainNameResolver() { try { InetAddress nn1Address = InetAddress.getByAddress(BYTE_ADDR_1); InetAddress nn2Address = InetAddress.getByAddress(BYTE_ADDR_2); addrs.put(DOMAIN, new InetAddress[]{nn1Address, nn2Address}); + ptrMap.put(nn1Address, FQDN_1); + ptrMap.put(nn2Address, FQDN_2); } catch (UnknownHostException e) { throw new RuntimeException(e); } @@ -61,6 +68,29 @@ public InetAddress[] getAllByDomainName(String domainName) return addrs.get(domainName); } + @Override + public String getHostnameByIP(InetAddress address) { + return ptrMap.containsKey(address) ? ptrMap.get(address) : null; + } + + @Override + public String[] getAllResolvedHostnameByDomainName( + String domainName, boolean useFQDN) throws UnknownHostException { + InetAddress[] addresses = getAllByDomainName(domainName); + String[] hosts = new String[addresses.length]; + if (useFQDN) { + for (int i = 0; i < hosts.length; i++) { + hosts[i] = this.ptrMap.get(addresses[i]); + } + } else { + for (int i = 0; i < hosts.length; i++) { + hosts[i] = addresses[i].getHostAddress(); + } + } + + return hosts; + } + @VisibleForTesting public void setAddressMap(Map addresses) { this.addrs = addresses; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java index 67904ee11b..1cd9018dc6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java @@ -291,6 +291,8 @@ interface Failover { String RESOLVE_ADDRESS_NEEDED_KEY = PREFIX + "resolve-needed"; boolean RESOLVE_ADDRESS_NEEDED_DEFAULT = false; String RESOLVE_SERVICE_KEY = PREFIX + "resolver.impl"; + String RESOLVE_ADDRESS_TO_FQDN = PREFIX + "useFQDN"; + boolean RESOLVE_ADDRESS_TO_FQDN_DEFAULT = true; } /** dfs.client.write configuration properties */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/AbstractNNFailoverProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/AbstractNNFailoverProxyProvider.java index 93452a3da8..e1e5fd09a8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/AbstractNNFailoverProxyProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/AbstractNNFailoverProxyProvider.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hdfs.server.namenode.ha; import java.io.IOException; -import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.URI; import java.util.ArrayList; @@ -180,7 +179,7 @@ protected List> getProxyAddresses(URI uri, String addressKey) { Collection addressesOfNns = addressesInNN.values(); try { - addressesOfNns = getResolvedAddressesIfNecessary(addressesOfNns, uri); + addressesOfNns = getResolvedHostsIfNecessary(addressesOfNns, uri); } catch (IOException e) { throw new RuntimeException(e); } @@ -209,7 +208,7 @@ protected List> getProxyAddresses(URI uri, String addressKey) { * @return The collection of resolved IP addresses. * @throws IOException If there are issues resolving the addresses. */ - Collection getResolvedAddressesIfNecessary( + Collection getResolvedHostsIfNecessary( Collection addressesOfNns, URI nameNodeUri) throws IOException { // 'host' here is usually the ID of the nameservice when address @@ -223,6 +222,11 @@ Collection getResolvedAddressesIfNecessary( // Early return is no resolve is necessary return addressesOfNns; } + // decide whether to access server by IP or by host name + String useFQDNKeyWithHost = + HdfsClientConfigKeys.Failover.RESOLVE_ADDRESS_TO_FQDN + "." + host; + boolean requireFQDN = conf.getBoolean(useFQDNKeyWithHost, + HdfsClientConfigKeys.Failover.RESOLVE_ADDRESS_TO_FQDN_DEFAULT); Collection addressOfResolvedNns = new ArrayList<>(); DomainNameResolver dnr = DomainNameResolverFactory.newInstance( @@ -232,12 +236,12 @@ Collection getResolvedAddressesIfNecessary( LOG.info("Namenode domain name will be resolved with {}", dnr.getClass().getName()); for (InetSocketAddress address : addressesOfNns) { - InetAddress[] resolvedAddresses = dnr.getAllByDomainName( - address.getHostName()); + String[] resolvedHostNames = dnr.getAllResolvedHostnameByDomainName( + address.getHostName(), requireFQDN); int port = address.getPort(); - for (InetAddress raddress : resolvedAddresses) { + for (String hostname : resolvedHostNames) { InetSocketAddress resolvedAddress = new InetSocketAddress( - raddress, port); + hostname, port); addressOfResolvedNns.add(resolvedAddress); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConfiguredFailoverProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConfiguredFailoverProxyProvider.java index 58922463d2..e3f34e3c66 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConfiguredFailoverProxyProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConfiguredFailoverProxyProvider.java @@ -142,7 +142,8 @@ public void setup() throws URISyntaxException { * Add more DNS related settings to the passed in configuration. * @param config Configuration file to add settings to. */ - private void addDNSSettings(Configuration config, boolean hostResolvable) { + private void addDNSSettings(Configuration config, + boolean hostResolvable, boolean useFQDN) { config.set( HdfsClientConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + "." + ns3, "nn"); String domain = hostResolvable @@ -163,6 +164,10 @@ private void addDNSSettings(Configuration config, boolean hostResolvable) { config.setBoolean( HdfsClientConfigKeys.Failover.RANDOM_ORDER + "." + ns3, true); + config.setBoolean( + HdfsClientConfigKeys.Failover.RESOLVE_ADDRESS_TO_FQDN + "." + ns3, + useFQDN + ); } /** @@ -250,17 +255,18 @@ public void testRandomGetProxy() throws Exception { nn1Count.get() + nn2Count.get() + nn3Count.get()); } - @Test - public void testResolveDomainNameUsingDNS() throws Exception { + private void testResolveDomainNameUsingDNS(boolean useFQDN) throws Exception { Configuration dnsConf = new Configuration(conf); - addDNSSettings(dnsConf, true); + addDNSSettings(dnsConf, true, useFQDN); // Mock ClientProtocol Map proxyMap = new HashMap<>(); final AtomicInteger nn1Count = addClientMock( - MockDomainNameResolver.BYTE_ADDR_1, proxyMap); + useFQDN ? MockDomainNameResolver.FQDN_1 : MockDomainNameResolver.ADDR_1, + proxyMap); final AtomicInteger nn2Count = addClientMock( - MockDomainNameResolver.BYTE_ADDR_2, proxyMap); + useFQDN ? MockDomainNameResolver.FQDN_2 : MockDomainNameResolver.ADDR_2, + proxyMap); // Get a client multiple times final Map proxyResults = new HashMap<>(); @@ -280,16 +286,18 @@ public void testResolveDomainNameUsingDNS() throws Exception { proxy.getStats(); } + String resolvedHost1 = useFQDN ? + MockDomainNameResolver.FQDN_1 : "/" + MockDomainNameResolver.ADDR_1; + String resolvedHost2 = useFQDN ? + MockDomainNameResolver.FQDN_2 : "/" + MockDomainNameResolver.ADDR_2; // Check we got the proper addresses assertEquals(2, proxyResults.size()); assertTrue( "nn1 wasn't returned: " + proxyResults, - proxyResults.containsKey( - "/" + MockDomainNameResolver.ADDR_1 + ":8020")); + proxyResults.containsKey(resolvedHost1 + ":8020")); assertTrue( "nn2 wasn't returned: " + proxyResults, - proxyResults.containsKey( - "/" + MockDomainNameResolver.ADDR_2 + ":8020")); + proxyResults.containsKey(resolvedHost2 + ":8020")); // Check that the Namenodes were invoked assertEquals(NUM_ITERATIONS, nn1Count.get() + nn2Count.get()); @@ -304,10 +312,18 @@ public void testResolveDomainNameUsingDNS() throws Exception { nn2Count.get() > 0); } + @Test + public void testResolveDomainNameUsingDNS() throws Exception { + // test resolving to IP + testResolveDomainNameUsingDNS(false); + // test resolving to FQDN + testResolveDomainNameUsingDNS(true); + } + @Test public void testResolveDomainNameUsingDNSUnknownHost() throws Exception { Configuration dnsConf = new Configuration(conf); - addDNSSettings(dnsConf, false); + addDNSSettings(dnsConf, false, false); Map proxyMap = new HashMap<>(); exception.expect(RuntimeException.class); @@ -321,19 +337,18 @@ public void testResolveDomainNameUsingDNSUnknownHost() throws Exception { /** * Add a ClientProtocol mock for the proxy. - * @param addr IP address for the destination. + * @param host host name for the destination. * @param proxyMap Map containing the client for each target address. * @return The counter for the number of calls to this target. * @throws Exception If the client cannot be created. */ private AtomicInteger addClientMock( - byte[] addr, Map proxyMap) - throws Exception { + String host, Map proxyMap) + throws Exception { final AtomicInteger counter = new AtomicInteger(0); - InetAddress inetAddr = InetAddress.getByAddress(addr); InetSocketAddress inetSockerAddr = - new InetSocketAddress(inetAddr, rpcPort); + new InetSocketAddress(host, rpcPort); final ClientProtocol cpMock = mock(ClientProtocol.class); when(cpMock.getStats()).thenAnswer(createAnswer(counter, 1)); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 8a8b55d0c1..bf4e7b9528 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -3798,6 +3798,20 @@ + + dfs.client.failover.resolver.useFQDN + true + + Determines whether the resolved result is fully qualified domain name instead + of pure IP address(es). The config name can be extended with an optional + nameservice ID (of form dfs.client.failover.resolver.impl[.nameservice]) to + configure specific nameservices when multiple nameservices exist. + In secure environment, this has to be enabled since Kerberos is using fqdn + in machine's principal therefore accessing servers by IP won't be recognized + by the KDC. + + + dfs.client.key.provider.cache.expiry 864000000