big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-14632. Reduce useless #getNumLiveDataNodes call in SafeModeMonitor. Contributed by He Xiaoqiao.

Browse Source
This commit is contained in:
Yiqun Lin 2019-07-10 10:53:34 +08:00
parent ac7a8accdf
commit 993dc8726b
4 changed files with 62 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java
View File

@ -309,16 +309,20 @@ String getSafeModeTip() {
}
}
int numLive = blockManager.getDatanodeManager().getNumLiveDataNodes();
if (numLive < datanodeThreshold) {
msg += String.format(
"The number of live datanodes %d needs an additional %d live "
+ "datanodes to reach the minimum number %d.%n",
numLive, (datanodeThreshold - numLive), datanodeThreshold);
if (datanodeThreshold > 0) {
int numLive = blockManager.getDatanodeManager().getNumLiveDataNodes();
if (numLive < datanodeThreshold) {
msg += String.format(
"The number of live datanodes %d needs an additional %d live "
+ "datanodes to reach the minimum number %d.%n",
numLive, (datanodeThreshold - numLive), datanodeThreshold);
} else {
msg += String.format("The number of live datanodes %d has reached "
+ "the minimum number %d. ",
numLive, datanodeThreshold);
}
} else {
msg += String.format("The number of live datanodes %d has reached "
+ "the minimum number %d. ",
numLive, datanodeThreshold);
msg += "The minimum number of live datanodes is not required. ";
}
if (getBytesInFuture() > 0) {

10
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
View File

@ -201,11 +201,11 @@ public void testInitializeReplQueuesEarly() throws Exception {
final NameNode nn = cluster.getNameNode();
String status = nn.getNamesystem().getSafemode();
assertEquals("Safe mode is ON. The reported blocks 0 needs additional " +
"14 blocks to reach the threshold 0.9990 of total blocks 15." + NEWLINE +
"The number of live datanodes 0 has reached the minimum number 0. " +
"Safe mode will be turned off automatically once the thresholds " +
"have been reached.", status);
assertEquals("Safe mode is ON. The reported blocks 0 needs additional "
+ "14 blocks to reach the threshold 0.9990 of total blocks 15."
+ NEWLINE + "The minimum number of live datanodes is not required. "
+ "Safe mode will be turned off automatically once the thresholds have "
+ "been reached.", status);
assertFalse("Mis-replicated block queues should not be initialized " +
"until threshold is crossed",
NameNodeAdapter.safeModeInitializedReplQueues(nn));

25
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java
View File

@ -468,6 +468,31 @@ public void testGetSafeModeTip() throws Exception {
assertTrue(tip.contains("Safe mode will be turned off automatically soon"));
}
/**
* Test get safe mode tip without minimum number of live datanodes required.
*/
@Test
public void testGetSafeModeTipsWithoutNumLiveDatanode() throws IOException {
Configuration conf = new HdfsConfiguration();
conf.setDouble(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY,
THRESHOLD);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY,
EXTENSION);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_MIN_DATANODES_KEY, 0);
NameNode.initMetrics(conf, NamenodeRole.NAMENODE);
BlockManager blockManager = spy(new BlockManager(fsn, false, conf));
BlockManagerSafeMode safeMode = new BlockManagerSafeMode(blockManager,
fsn, false, conf);
safeMode.activate(BLOCK_TOTAL);
String tip = safeMode.getSafeModeTip();
assertTrue(tip.contains("The minimum number of live datanodes is not "
+ "required."));
}
/**
* Test get safe mode tip in case of blocks with future GS.
*/

29
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
View File

@ -498,13 +498,22 @@ private static void assertSafeMode(NameNode nn, int safe, int total,
int numNodes, int nodeThresh) {
String status = nn.getNamesystem().getSafemode();
if (safe == total) {
assertTrue("Bad safemode status: '" + status + "'",
status.startsWith(
"Safe mode is ON. The reported blocks " + safe + " has reached the "
+ "threshold 0.9990 of total blocks " + total + ". The number of "
+ "live datanodes " + numNodes + " has reached the minimum number "
+ nodeThresh + ". In safe mode extension. "
+ "Safe mode will be turned off automatically"));
if (nodeThresh == 0) {
assertTrue("Bad safemode status: '" + status + "'",
status.startsWith("Safe mode is ON. The reported blocks " + safe
+ " has reached the " + "threshold 0.9990 of total blocks "
+ total + ". The minimum number of live datanodes is not "
+ "required. In safe mode extension. Safe mode will be turned "
+ "off automatically"));
} else {
assertTrue("Bad safemode status: '" + status + "'",
status.startsWith(
"Safe mode is ON. The reported blocks " + safe + " has reached "
+ "the threshold 0.9990 of total blocks " + total + ". The "
+ "number of live datanodes " + numNodes + " has reached "
+ "the minimum number " + nodeThresh + ". In safe mode "
+ "extension. Safe mode will be turned off automatically"));
}
} else {
int additional = (int) (total * 0.9990) - safe;
assertTrue("Bad safemode status: '" + status + "'",
@ -573,9 +582,9 @@ public void testBlocksRemovedWhileInSafeModeEditsArriveFirst() throws Exception
assertTrue("Bad safemode status: '" + status + "'",
status.startsWith(
"Safe mode is ON. The reported blocks 10 has reached the threshold "
+ "0.9990 of total blocks 10. The number of live datanodes 3 has "
+ "reached the minimum number 0. In safe mode extension. "
+ "Safe mode will be turned off automatically"));
+ "0.9990 of total blocks 10. The minimum number of live datanodes is "
+ "not required. In safe mode extension. Safe mode will be turned off "
+ "automatically"));
// Delete those blocks while the SBN is in safe mode.
// Immediately roll the edit log before the actual deletions are sent