big-data/hadoop
big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Projects Wiki Activity

HDFS-15920.Solve the problem that the value of SafeModeMonitor#RECHECK_INTERVAL can be configured. (#2831). Contributed by JiangHua Zhu.

Browse Source 
Signed-off-by: Ayush Saxena <ayushsaxena@apache.org>
This commit is contained in:
jianghuazhu 2021-09-14 16:54:36 +08:00 committed by GitHub
parent c4c5883d8b
commit 1032104b9a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 42 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
View File

@ -207,6 +207,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
"dfs.namenode.replqueue.threshold-pct"; "dfs.namenode.replqueue.threshold-pct";
public static final String DFS_NAMENODE_SAFEMODE_MIN_DATANODES_KEY = "dfs.namenode.safemode.min.datanodes"; public static final String DFS_NAMENODE_SAFEMODE_MIN_DATANODES_KEY = "dfs.namenode.safemode.min.datanodes";
public static final int DFS_NAMENODE_SAFEMODE_MIN_DATANODES_DEFAULT = 0; public static final int DFS_NAMENODE_SAFEMODE_MIN_DATANODES_DEFAULT = 0;
public static final String DFS_NAMENODE_SAFEMODE_RECHECK_INTERVAL_KEY =
"dfs.namenode.safemode.recheck.interval";
public static final long DFS_NAMENODE_SAFEMODE_RECHECK_INTERVAL_DEFAULT =
1000;
public static final String DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY = public static final String DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY =
HdfsClientConfigKeys.DeprecatedKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY; HdfsClientConfigKeys.DeprecatedKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY;
public static final String DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_DEFAULT = public static final String DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_DEFAULT =

22
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java
View File

@ -109,7 +109,7 @@ class BlockManagerSafeMode {
/** Timestamp of the safe mode initialized. */ /** Timestamp of the safe mode initialized. */
private long startTime; private long startTime;
/** the safe mode monitor thread. */ /** the safe mode monitor thread. */
private final Daemon smmthread = new Daemon(new SafeModeMonitor()); private final Daemon smmthread;
/** time of the last status printout */ /** time of the last status printout */
private long lastStatusReport; private long lastStatusReport;
@ -156,6 +156,7 @@ class BlockManagerSafeMode {
MILLISECONDS); MILLISECONDS);
this.inRollBack = isInRollBackMode(NameNode.getStartupOption(conf)); this.inRollBack = isInRollBackMode(NameNode.getStartupOption(conf));
this.smmthread = new Daemon(new SafeModeMonitor(conf));
LOG.info("{} = {}", DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY, threshold); LOG.info("{} = {}", DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY, threshold);
LOG.info("{} = {}", DFS_NAMENODE_SAFEMODE_MIN_DATANODES_KEY, LOG.info("{} = {}", DFS_NAMENODE_SAFEMODE_MIN_DATANODES_KEY,
@ -638,9 +639,22 @@ class BlockManagerSafeMode {
* Periodically check whether it is time to leave safe mode. * Periodically check whether it is time to leave safe mode.
* This thread starts when the threshold level is reached. * This thread starts when the threshold level is reached.
*/ */
private class SafeModeMonitor implements Runnable { final private class SafeModeMonitor implements Runnable {
/** Interval in msec for checking safe mode. */ /** Interval in msec for checking safe mode. */
private static final long RECHECK_INTERVAL = 1000; private long recheckInterval;
private SafeModeMonitor(Configuration conf) {
recheckInterval = conf.getLong(
DFSConfigKeys.DFS_NAMENODE_SAFEMODE_RECHECK_INTERVAL_KEY,
DFSConfigKeys.DFS_NAMENODE_SAFEMODE_RECHECK_INTERVAL_DEFAULT);
if (recheckInterval < 1) {
LOG.warn("Invalid value for " +
DFSConfigKeys.DFS_NAMENODE_SAFEMODE_RECHECK_INTERVAL_DEFAULT +
".Should be greater than 0, but is {}", recheckInterval);
recheckInterval = DFSConfigKeys.DFS_NAMENODE_SAFEMODE_RECHECK_INTERVAL_DEFAULT;
}
LOG.info("Using {} as SafeModeMonitor Interval", recheckInterval);
}
@Override @Override
public void run() { public void run() {
@ -660,7 +674,7 @@ class BlockManagerSafeMode {
} }
try { try {
Thread.sleep(RECHECK_INTERVAL); Thread.sleep(recheckInterval);
} catch (InterruptedException ignored) { } catch (InterruptedException ignored) {
} }
} }

8
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
View File

@ -1011,6 +1011,14 @@
</description> </description>
</property> </property>
<property>
<name>dfs.namenode.safemode.recheck.interval</name>
<value>1000</value>
<description>
Interval in msec for checking safe mode.
</description>
</property>
<property> <property>
<name>dfs.namenode.safemode.extension</name> <name>dfs.namenode.safemode.extension</name>
<value>30000</value> <value>30000</value>

12
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java
View File

@ -230,6 +230,18 @@ public class TestBlockManagerSafeMode {
assertEquals(BMSafeModeStatus.OFF, getSafeModeStatus()); assertEquals(BMSafeModeStatus.OFF, getSafeModeStatus());
} }
@Test(timeout = 20000)
public void testCheckSafeMode9() throws Exception {
Configuration conf = new HdfsConfiguration();
conf.setLong(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_RECHECK_INTERVAL_KEY, 3000);
GenericTestUtils.LogCapturer logs =
GenericTestUtils.LogCapturer.captureLogs(BlockManagerSafeMode.LOG);
BlockManagerSafeMode blockManagerSafeMode = new BlockManagerSafeMode(bm,
fsn, true, conf);
String content = logs.getOutput();
assertTrue(content.contains("Using 3000 as SafeModeMonitor Interval"));
}
/** /**
* Test that the block safe increases up to block threshold. * Test that the block safe increases up to block threshold.
* *