From 07c8b69f7cccae98be6e4d338baea4f0a58610f7 Mon Sep 17 00:00:00 2001 From: dannytbecker <43830149+dannytbecker@users.noreply.github.com> Date: Wed, 6 Sep 2023 09:17:12 -0700 Subject: [PATCH] HDFS-17167. Add config to startup NameNode as Observer (#6013) --- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 + .../hdfs/server/namenode/BackupNode.java | 2 +- .../hadoop/hdfs/server/namenode/NameNode.java | 14 ++++- .../src/main/resources/hdfs-default.xml | 9 +++ .../server/namenode/ha/TestObserverNode.java | 60 +++++++++++++++++++ 5 files changed, 83 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 7c6683dfc1..f0450b0778 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -1177,6 +1177,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_NAMENODE_PLUGINS_KEY = "dfs.namenode.plugins"; public static final String DFS_WEB_UGI_KEY = "dfs.web.ugi"; public static final String DFS_NAMENODE_STARTUP_KEY = "dfs.namenode.startup"; + public static final String DFS_NAMENODE_OBSERVER_ENABLED_KEY = "dfs.namenode.observer.enabled"; + public static final boolean DFS_NAMENODE_OBSERVER_ENABLED_DEFAULT = false; public static final String DFS_DATANODE_KEYTAB_FILE_KEY = "dfs.datanode.keytab.file"; public static final String DFS_DATANODE_KERBEROS_PRINCIPAL_KEY = HdfsClientConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java index 105f172812..a3d9746be1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupNode.java @@ -435,7 +435,7 @@ protected String getNameServiceId(Configuration conf) { } @Override - protected HAState createHAState(StartupOption startOpt) { + protected HAState createHAState(Configuration conf) { return new BackupState(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index 4aa81152fa..388f06f0e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -163,6 +163,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_METRICS_LOGGER_PERIOD_SECONDS_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_METRICS_LOGGER_PERIOD_SECONDS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_OBSERVER_ENABLED_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_OBSERVER_ENABLED_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_PLUGINS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_BIND_HOST_KEY; @@ -1125,7 +1127,7 @@ protected NameNode(Configuration conf, NamenodeRole role) + " this namenode/service.", clientNamenodeAddress); } this.haEnabled = HAUtil.isHAEnabled(conf, nsId); - state = createHAState(getStartupOption(conf)); + state = createHAState(conf); this.allowStaleStandbyReads = HAUtil.shouldAllowStandbyReads(conf); this.haContext = createHAContext(); try { @@ -1161,11 +1163,17 @@ private void stopAtException(Exception e){ } } - protected HAState createHAState(StartupOption startOpt) { + protected HAState createHAState(Configuration conf) { + StartupOption startOpt = getStartupOption(conf); if (!haEnabled || startOpt == StartupOption.UPGRADE || startOpt == StartupOption.UPGRADEONLY) { return ACTIVE_STATE; - } else if (startOpt == StartupOption.OBSERVER) { + } else if (conf.getBoolean(DFS_NAMENODE_OBSERVER_ENABLED_KEY, + DFS_NAMENODE_OBSERVER_ENABLED_DEFAULT) + || startOpt == StartupOption.OBSERVER) { + // Set Observer state using config instead of startup option + // This allows other startup options to be used when starting observer. + // e.g. rollingUpgrade return OBSERVER_STATE; } else { return STANDBY_STATE; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 8d85d7cc3c..c8f5993adb 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -2801,6 +2801,15 @@ + + dfs.namenode.observer.enabled + false + + This causes NameNode on startup to become an observer node if + set to true, otherwise startup is no different. + + + dfs.namenode.enable.retrycache true diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java index 48f2cb185e..a293cb4d17 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestObserverNode.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode.ha; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_OBSERVER_ENABLED_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_STATE_CONTEXT_ENABLED_KEY; import static org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter.getServiceState; import static org.apache.hadoop.hdfs.server.namenode.ha.ObserverReadProxyProvider.*; @@ -215,6 +216,65 @@ public void testSimpleRead() throws Exception { assertSentTo(0); } + @Test + public void testConfigStartup() throws Exception { + int nnIdx = dfsCluster.getNumNameNodes() - 1; + + // Transition all current observers to standby + for (int i = 0; i < dfsCluster.getNumNameNodes(); i++) { + if (dfsCluster.getNameNode(i).isObserverState()) { + dfsCluster.transitionToStandby(i); + } + } + + // Confirm that the namenode at nnIdx is standby + assertTrue("The NameNode is observer despite being transitioned to standby", + dfsCluster.getNameNode(nnIdx).isStandbyState()); + + // Restart the NameNode with observer startup option as false + dfsCluster.getConfiguration(nnIdx) + .setBoolean(DFS_NAMENODE_OBSERVER_ENABLED_KEY, false); + dfsCluster.restartNameNode(nnIdx); + + // Verify that the NameNode is not in Observer state + dfsCluster.waitNameNodeUp(nnIdx); + assertTrue("The NameNode started as Observer despite " + + DFS_NAMENODE_OBSERVER_ENABLED_KEY + " being false", + dfsCluster.getNameNode(nnIdx).isStandbyState()); + + dfs.mkdir(testPath, FsPermission.getDefault()); + assertSentTo(0); + + // The first request goes to the active because it has not refreshed yet; + // the second would go to the observer if it was not in standby + dfsCluster.rollEditLogAndTail(0); + dfs.getFileStatus(testPath); + dfs.getFileStatus(testPath); + assertSentTo(0); + + Path testPath2 = new Path(testPath, "test2"); + // Restart the NameNode with the observer startup option as true + dfsCluster.getConfiguration(nnIdx) + .setBoolean(DFS_NAMENODE_OBSERVER_ENABLED_KEY, true); + dfsCluster.restartNameNode(nnIdx); + + // Check that the NameNode is in Observer state + dfsCluster.waitNameNodeUp(nnIdx); + assertTrue("The NameNode did not start as Observer despite " + + DFS_NAMENODE_OBSERVER_ENABLED_KEY + " being true", + dfsCluster.getNameNode(nnIdx).isObserverState()); + + dfs.mkdir(testPath2, FsPermission.getDefault()); + assertSentTo(0); + + // The first request goes to the active because it has not refreshed yet; + // the second will properly go to the observer + dfsCluster.rollEditLogAndTail(0); + dfs.getFileStatus(testPath2); + dfs.getFileStatus(testPath2); + assertSentTo(nnIdx); + } + @Test public void testFailover() throws Exception { Path testPath2 = new Path(testPath, "test2");