From b53cae0ffb1b383824a86185ef23f698719d7a6d Mon Sep 17 00:00:00 2001 From: LeonGao Date: Wed, 25 Aug 2021 17:40:12 -0700 Subject: [PATCH] HDFS-16157. Support configuring DNS record to get list of journal nodes contributed by Leon Gao. (#3284) * Add DNS resolution for QJM * Add log * Resolve comments * checkstyle * typo --- .../ha/TestZKFailoverControllerStress.java | 2 +- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 11 +++++ .../java/org/apache/hadoop/hdfs/DFSUtil.java | 6 +-- .../qjournal/client/QuorumJournalManager.java | 2 +- .../qjournal/server/JournalNodeSyncer.java | 2 +- .../hadoop/hdfs/server/common/Util.java | 46 +++++++++++++++---- .../src/main/resources/hdfs-default.xml | 19 ++++++++ .../markdown/HDFSHighAvailabilityWithQJM.md | 9 ++++ .../client/TestQuorumJournalManager.java | 30 ++++++++++++ 9 files changed, 113 insertions(+), 14 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestZKFailoverControllerStress.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestZKFailoverControllerStress.java index 4dcc74b86d..1fd339bfc3 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestZKFailoverControllerStress.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestZKFailoverControllerStress.java @@ -128,7 +128,7 @@ public void testRandomHealthAndDisconnects() throws Exception { // Mockito errors if the HM calls the proxy in the middle of // setting up the mock. cluster.start(); - + long st = Time.now(); while (Time.now() - st < runFor) { cluster.getTestContext().checkException(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 808ecfbe0c..ac0363f1c4 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -645,6 +645,17 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_NAMENODE_EDITS_PLUGIN_PREFIX = "dfs.namenode.edits.journal-plugin"; public static final String DFS_NAMENODE_EDITS_DIR_REQUIRED_KEY = "dfs.namenode.edits.dir.required"; public static final String DFS_NAMENODE_EDITS_DIR_DEFAULT = "file:///tmp/hadoop/dfs/name"; + + public static final String + DFS_NAMENODE_EDITS_QJOURNALS_RESOLUTION_ENABLED = + "dfs.namenode.edits.qjournals.resolution-enabled"; + public static final boolean + DFS_NAMENODE_EDITS_QJOURNALS_RESOLUTION_ENABLED_DEFAULT = false; + + public static final String + DFS_NAMENODE_EDITS_QJOURNALS_RESOLUTION_RESOLVER_IMPL = + "dfs.namenode.edits.qjournals.resolver.impl"; + public static final String DFS_METRICS_SESSION_ID_KEY = HdfsClientConfigKeys.DeprecatedKeys.DFS_METRICS_SESSION_ID_KEY; public static final String DFS_METRICS_PERCENTILES_INTERVALS_KEY = "dfs.metrics.percentiles.intervals"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index 34cd2558e9..36e76c89e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -490,7 +490,7 @@ public static Set getJournalNodeAddresses( " to append it with namenodeId"); URI uri = new URI(journalsUri); List socketAddresses = Util. - getAddressesList(uri); + getAddressesList(uri, conf); for (InetSocketAddress is : socketAddresses) { journalNodeList.add(is.getHostName()); } @@ -501,7 +501,7 @@ public static Set getJournalNodeAddresses( } else { URI uri = new URI(journalsUri); List socketAddresses = Util. - getAddressesList(uri); + getAddressesList(uri, conf); for (InetSocketAddress is : socketAddresses) { journalNodeList.add(is.getHostName()); } @@ -512,7 +512,7 @@ public static Set getJournalNodeAddresses( return journalNodeList; } else { URI uri = new URI(journalsUri); - List socketAddresses = Util.getAddressesList(uri); + List socketAddresses = Util.getAddressesList(uri, conf); for (InetSocketAddress is : socketAddresses) { journalNodeList.add(is.getHostName()); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java index 06a22d65b3..d2846be11f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java @@ -414,7 +414,7 @@ static List createLoggers(Configuration conf, String nameServiceId) throws IOException { List ret = Lists.newArrayList(); - List addrs = Util.getAddressesList(uri); + List addrs = Util.getAddressesList(uri, conf); if (addrs.size() % 2 == 0) { LOG.warn("Quorum journal URI '" + uri + "' has an even number " + "of Journal Nodes specified. This is not recommended!"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java index ce580cc95a..ff46aa751d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeSyncer.java @@ -315,7 +315,7 @@ private List getJournalAddrList(String uriStr) throws IOException { URI uri = new URI(uriStr); return Util.getLoggerAddresses(uri, - Sets.newHashSet(jn.getBoundIpcAddress())); + Sets.newHashSet(jn.getBoundIpcAddress()), conf); } private void getMissingLogSegments(List thisJournalEditLogs, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java index ccfb62c86c..1f3119b3b9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Util.java @@ -43,6 +43,8 @@ import org.apache.hadoop.hdfs.server.namenode.ImageServlet; import org.apache.hadoop.hdfs.util.DataTransferThrottler; import org.apache.hadoop.io.MD5Hash; +import org.apache.hadoop.net.DomainNameResolver; +import org.apache.hadoop.net.DomainNameResolverFactory; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.client.AuthenticationException; @@ -361,7 +363,7 @@ private static MD5Hash parseMD5Header(HttpURLConnection connection) { return (header != null) ? new MD5Hash(header) : null; } - public static List getAddressesList(URI uri) + public static List getAddressesList(URI uri, Configuration conf) throws IOException{ String authority = uri.getAuthority(); Preconditions.checkArgument(authority != null && !authority.isEmpty(), @@ -372,21 +374,49 @@ public static List getAddressesList(URI uri) parts[i] = parts[i].trim(); } + boolean resolveNeeded = conf.getBoolean( + DFSConfigKeys.DFS_NAMENODE_EDITS_QJOURNALS_RESOLUTION_ENABLED, + DFSConfigKeys.DFS_NAMENODE_EDITS_QJOURNALS_RESOLUTION_ENABLED_DEFAULT); + DomainNameResolver dnr = DomainNameResolverFactory.newInstance( + conf, + DFSConfigKeys.DFS_NAMENODE_EDITS_QJOURNALS_RESOLUTION_RESOLVER_IMPL); + List addrs = Lists.newArrayList(); for (String addr : parts) { - InetSocketAddress isa = NetUtils.createSocketAddr( - addr, DFSConfigKeys.DFS_JOURNALNODE_RPC_PORT_DEFAULT); - if (isa.isUnresolved()) { - throw new UnknownHostException(addr); + if (resolveNeeded) { + LOG.info("Resolving journal address: " + addr); + InetSocketAddress isa = NetUtils.createSocketAddr( + addr, DFSConfigKeys.DFS_JOURNALNODE_RPC_PORT_DEFAULT); + // Get multiple hostnames from domain name if needed, + // for example multiple hosts behind a DNS entry. + int port = isa.getPort(); + // QJM should just use FQDN + String[] hostnames = dnr + .getAllResolvedHostnameByDomainName(isa.getHostName(), true); + if (hostnames.length == 0) { + throw new UnknownHostException(addr); + } + for (String h : hostnames) { + addrs.add(NetUtils.createSocketAddr( + h + ":" + port, + DFSConfigKeys.DFS_JOURNALNODE_RPC_PORT_DEFAULT) + ); + } + } else { + InetSocketAddress isa = NetUtils.createSocketAddr( + addr, DFSConfigKeys.DFS_JOURNALNODE_RPC_PORT_DEFAULT); + if (isa.isUnresolved()) { + throw new UnknownHostException(addr); + } + addrs.add(isa); } - addrs.add(isa); } return addrs; } public static List getLoggerAddresses(URI uri, - Set addrsToExclude) throws IOException { - List addrsList = getAddressesList(uri); + Set addrsToExclude, Configuration conf) throws IOException { + List addrsList = getAddressesList(uri, conf); addrsList.removeAll(addrsToExclude); return addrsList; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 5818ae1812..642057fe00 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -502,6 +502,25 @@ org.apache.hadoop.hdfs.qjournal.client.QuorumJournalManager + + dfs.namenode.edits.qjournals.resolution-enabled + false + + Determines if the given qjournals address is a domain name which needs to + be resolved. + This is used by namenode to resolve qjournals. + + + + + dfs.namenode.edits.qjournals.resolver.impl + + + Qjournals resolver implementation used by namenode. + Effective with dfs.namenode.edits.qjournals.resolution-enabled on. + + + dfs.permissions.enabled true diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md index eaa1a86db4..4bb5971107 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md @@ -194,6 +194,15 @@ The order in which you set these configurations is unimportant, but the values y qjournal://node1.example.com:8485;node2.example.com:8485;node3.example.com:8485/mycluster + You can also configure journal nodes by setting up dns round-robin record to avoid hardcoded names: + + + dfs.namenode.edits.qjournals.resolution-enabled + true + + + This will require you to configure multiple IPs behind one dns record on the host level, for example round robin DNS. + * **dfs.client.failover.proxy.provider.[nameservice ID]** - the Java class that HDFS clients use to contact the Active NameNode Configure the name of the Java class which will be used by the DFS Client to diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java index c4760a0a6e..5666ae58b0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java @@ -33,13 +33,17 @@ import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; +import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeoutException; +import org.apache.hadoop.hdfs.server.common.Util; +import org.apache.hadoop.net.MockDomainNameResolver; import org.apache.hadoop.util.Lists; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -1123,6 +1127,32 @@ public void testSelectViaRpcAfterJNRestart() throws Exception { Mockito.verify(logger, Mockito.times(1)).getEditLogManifest(1, true); } } + + @Test + public void testGetJournalAddressListWithResolution() throws Exception { + Configuration configuration = new Configuration(); + configuration.setBoolean( + DFSConfigKeys.DFS_NAMENODE_EDITS_QJOURNALS_RESOLUTION_ENABLED, true); + configuration.set( + DFSConfigKeys.DFS_NAMENODE_EDITS_QJOURNALS_RESOLUTION_RESOLVER_IMPL, + MockDomainNameResolver.class.getName()); + + URI uriWithDomain = URI.create("qjournal://" + + MockDomainNameResolver.DOMAIN + ":1234" + "/testns"); + List result = Util.getAddressesList(uriWithDomain, configuration); + assertEquals(2, result.size()); + assertEquals(new InetSocketAddress(MockDomainNameResolver.FQDN_1, 1234), result.get(0)); + assertEquals(new InetSocketAddress(MockDomainNameResolver.FQDN_2, 1234), result.get(1)); + + uriWithDomain = URI.create("qjournal://" + + MockDomainNameResolver.UNKNOW_DOMAIN + ":1234" + "/testns"); + try{ + Util.getAddressesList(uriWithDomain, configuration); + fail("Should throw unknown host exception."); + } catch (UnknownHostException e) { + // expected + } + } private QuorumJournalManager createSpyingQJM() throws IOException, URISyntaxException {