big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-15561. RBF: Remove NPE when local namenode is not configured (#2954). Contributed by Fengnan Li.

Browse Source 
Reviewed-by: He Xiaoqiao <hexiaoqiao@apache.org>
This commit is contained in:
lfengnan 2021-04-29 06:14:18 -07:00 committed by GitHub
parent b2e54762a4
commit 552e9dcc6c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 55 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
View File

@ -550,14 +550,20 @@ public void verifyToken(DelegationTokenIdentifier tokenId, byte[] password)
*
* @return Updater of the status for the local Namenode.
*/
protected NamenodeHeartbeatService createLocalNamenodeHeartbeatService() {
@VisibleForTesting
public NamenodeHeartbeatService createLocalNamenodeHeartbeatService() {
// Detect NN running in this machine
String nsId = DFSUtil.getNamenodeNameServiceId(conf);
if (nsId == null) {
LOG.error("Cannot find local nameservice id");
return null;
}
String nnId = null;
if (HAUtil.isHAEnabled(conf, nsId)) {
nnId = HAUtil.getNameNodeId(conf, nsId);
if (nnId == null) {
LOG.error("Cannot find namenode id for local {}", nsId);
return null;
}
}
@ -787,4 +793,13 @@ public RouterAdminServer getAdminServer() {
return adminServer;
}
/**
* Set router configuration.
* @param conf
*/
@VisibleForTesting
public void setConf(Configuration conf) {
this.conf = conf;
}
}

38
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterNamenodeHeartbeat.java
View File

@ -17,15 +17,21 @@
*/
package org.apache.hadoop.hdfs.server.federation.router;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICES;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY;
import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMENODES;
import static org.apache.hadoop.hdfs.server.federation.FederationTestUtils.NAMESERVICES;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.server.federation.MockResolver;
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster;
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext;
@ -105,6 +111,38 @@ public void testNamenodeHeartbeatService() throws IOException {
server.close();
}
@Test
public void testLocalNamenodeHeartbeatService() throws IOException {
Router router = new Router();
Configuration conf = new Configuration();
assertEquals(null, DFSUtil.getNamenodeNameServiceId(conf));
// case 1: no local nn is configured
router.setConf(conf);
assertNull(router.createLocalNamenodeHeartbeatService());
// case 2: local nn is configured
conf.set(DFS_NAMESERVICES, "ns1");
assertEquals("ns1", DFSUtil.getNamenodeNameServiceId(conf));
conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, "ns1"),
"nn1,nn2");
conf.set(DFSUtil.addKeySuffixes(
DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn1"),
"localhost:8020");
conf.set(DFSUtil.addKeySuffixes(
DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn2"),
"ns1-nn2.example.com:8020");
router.setConf(conf);
NamenodeHeartbeatService heartbeatService =
router.createLocalNamenodeHeartbeatService();
assertNotNull(heartbeatService);
// we have to start the service to get the serviceAddress assigned
heartbeatService.init(conf);
assertEquals("ns1-nn1:localhost:8020",
heartbeatService.getNamenodeDesc());
heartbeatService.stop();
}
@Test
public void testHearbeat() throws InterruptedException, IOException {

1
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterWebHdfsMethods.java
View File

@ -61,6 +61,7 @@ public static void globalSetUp() throws Exception {
.admin()
.build();
cluster.addRouterOverrides(conf);
cluster.setIndependentDNs();
cluster.startCluster();
cluster.startRouters();
cluster.waitClusterUp();