From 993dc8726b7d40ac832ae3e23b64e8541b62c4bd Mon Sep 17 00:00:00 2001 From: Yiqun Lin Date: Wed, 10 Jul 2019 10:53:34 +0800 Subject: [PATCH] HDFS-14632. Reduce useless #getNumLiveDataNodes call in SafeModeMonitor. Contributed by He Xiaoqiao. --- .../blockmanagement/BlockManagerSafeMode.java | 22 ++++++++------ .../org/apache/hadoop/hdfs/TestSafeMode.java | 10 +++---- .../TestBlockManagerSafeMode.java | 25 ++++++++++++++++ .../server/namenode/ha/TestHASafeMode.java | 29 ++++++++++++------- 4 files changed, 62 insertions(+), 24 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java index 1d7e69bacc..3f59b64d0c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java @@ -309,16 +309,20 @@ String getSafeModeTip() { } } - int numLive = blockManager.getDatanodeManager().getNumLiveDataNodes(); - if (numLive < datanodeThreshold) { - msg += String.format( - "The number of live datanodes %d needs an additional %d live " - + "datanodes to reach the minimum number %d.%n", - numLive, (datanodeThreshold - numLive), datanodeThreshold); + if (datanodeThreshold > 0) { + int numLive = blockManager.getDatanodeManager().getNumLiveDataNodes(); + if (numLive < datanodeThreshold) { + msg += String.format( + "The number of live datanodes %d needs an additional %d live " + + "datanodes to reach the minimum number %d.%n", + numLive, (datanodeThreshold - numLive), datanodeThreshold); + } else { + msg += String.format("The number of live datanodes %d has reached " + + "the minimum number %d. ", + numLive, datanodeThreshold); + } } else { - msg += String.format("The number of live datanodes %d has reached " - + "the minimum number %d. ", - numLive, datanodeThreshold); + msg += "The minimum number of live datanodes is not required. "; } if (getBytesInFuture() > 0) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java index 0fde81ecdf..7fbf222e7a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java @@ -201,11 +201,11 @@ public void testInitializeReplQueuesEarly() throws Exception { final NameNode nn = cluster.getNameNode(); String status = nn.getNamesystem().getSafemode(); - assertEquals("Safe mode is ON. The reported blocks 0 needs additional " + - "14 blocks to reach the threshold 0.9990 of total blocks 15." + NEWLINE + - "The number of live datanodes 0 has reached the minimum number 0. " + - "Safe mode will be turned off automatically once the thresholds " + - "have been reached.", status); + assertEquals("Safe mode is ON. The reported blocks 0 needs additional " + + "14 blocks to reach the threshold 0.9990 of total blocks 15." + + NEWLINE + "The minimum number of live datanodes is not required. " + + "Safe mode will be turned off automatically once the thresholds have " + + "been reached.", status); assertFalse("Mis-replicated block queues should not be initialized " + "until threshold is crossed", NameNodeAdapter.safeModeInitializedReplQueues(nn)); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java index fd224ea3df..5cf094d49a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java @@ -468,6 +468,31 @@ public void testGetSafeModeTip() throws Exception { assertTrue(tip.contains("Safe mode will be turned off automatically soon")); } + /** + * Test get safe mode tip without minimum number of live datanodes required. + */ + @Test + public void testGetSafeModeTipsWithoutNumLiveDatanode() throws IOException { + Configuration conf = new HdfsConfiguration(); + conf.setDouble(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY, + THRESHOLD); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, + EXTENSION); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_MIN_DATANODES_KEY, 0); + + NameNode.initMetrics(conf, NamenodeRole.NAMENODE); + + BlockManager blockManager = spy(new BlockManager(fsn, false, conf)); + + BlockManagerSafeMode safeMode = new BlockManagerSafeMode(blockManager, + fsn, false, conf); + safeMode.activate(BLOCK_TOTAL); + String tip = safeMode.getSafeModeTip(); + + assertTrue(tip.contains("The minimum number of live datanodes is not " + + "required.")); + } + /** * Test get safe mode tip in case of blocks with future GS. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java index 3f1a979d1d..4647dd78d9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java @@ -498,13 +498,22 @@ private static void assertSafeMode(NameNode nn, int safe, int total, int numNodes, int nodeThresh) { String status = nn.getNamesystem().getSafemode(); if (safe == total) { - assertTrue("Bad safemode status: '" + status + "'", - status.startsWith( - "Safe mode is ON. The reported blocks " + safe + " has reached the " - + "threshold 0.9990 of total blocks " + total + ". The number of " - + "live datanodes " + numNodes + " has reached the minimum number " - + nodeThresh + ". In safe mode extension. " - + "Safe mode will be turned off automatically")); + if (nodeThresh == 0) { + assertTrue("Bad safemode status: '" + status + "'", + status.startsWith("Safe mode is ON. The reported blocks " + safe + + " has reached the " + "threshold 0.9990 of total blocks " + + total + ". The minimum number of live datanodes is not " + + "required. In safe mode extension. Safe mode will be turned " + + "off automatically")); + } else { + assertTrue("Bad safemode status: '" + status + "'", + status.startsWith( + "Safe mode is ON. The reported blocks " + safe + " has reached " + + "the threshold 0.9990 of total blocks " + total + ". The " + + "number of live datanodes " + numNodes + " has reached " + + "the minimum number " + nodeThresh + ". In safe mode " + + "extension. Safe mode will be turned off automatically")); + } } else { int additional = (int) (total * 0.9990) - safe; assertTrue("Bad safemode status: '" + status + "'", @@ -573,9 +582,9 @@ public void testBlocksRemovedWhileInSafeModeEditsArriveFirst() throws Exception assertTrue("Bad safemode status: '" + status + "'", status.startsWith( "Safe mode is ON. The reported blocks 10 has reached the threshold " - + "0.9990 of total blocks 10. The number of live datanodes 3 has " - + "reached the minimum number 0. In safe mode extension. " - + "Safe mode will be turned off automatically")); + + "0.9990 of total blocks 10. The minimum number of live datanodes is " + + "not required. In safe mode extension. Safe mode will be turned off " + + "automatically")); // Delete those blocks while the SBN is in safe mode. // Immediately roll the edit log before the actual deletions are sent