big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-11095. BlockManagerSafeMode should respect extension period default config value (30s). Contributed by Mingliang Liu

Browse Source
This commit is contained in:
Mingliang Liu 2016-11-02 13:01:54 -07:00
parent f2800f523d
commit 9b0c17f852
3 changed files with 32 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerSafeMode.java
View File

@ -35,12 +35,16 @@
import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.util.Daemon; import org.apache.hadoop.util.Daemon;
import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REPL_QUEUE_THRESHOLD_PCT_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REPL_QUEUE_THRESHOLD_PCT_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SAFEMODE_MIN_DATANODES_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SAFEMODE_MIN_DATANODES_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SAFEMODE_MIN_DATANODES_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SAFEMODE_MIN_DATANODES_KEY;
@ -96,7 +100,8 @@ enum BMSafeModeStatus {
private long blockReplQueueThreshold; private long blockReplQueueThreshold;
/** How long (in ms) is the extension period. */ /** How long (in ms) is the extension period. */
private final int extension; @VisibleForTesting
final long extension;
/** Timestamp of the first time when thresholds are met. */ /** Timestamp of the first time when thresholds are met. */
private final AtomicLong reachedTime = new AtomicLong(); private final AtomicLong reachedTime = new AtomicLong();
/** Timestamp of the safe mode initialized. */ /** Timestamp of the safe mode initialized. */
@ -143,7 +148,9 @@ enum BMSafeModeStatus {
this.replQueueThreshold = this.replQueueThreshold =
conf.getFloat(DFS_NAMENODE_REPL_QUEUE_THRESHOLD_PCT_KEY, conf.getFloat(DFS_NAMENODE_REPL_QUEUE_THRESHOLD_PCT_KEY,
(float) threshold); (float) threshold);
this.extension = conf.getInt(DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, 0); this.extension = conf.getTimeDuration(DFS_NAMENODE_SAFEMODE_EXTENSION_KEY,
DFS_NAMENODE_SAFEMODE_EXTENSION_DEFAULT,
MILLISECONDS);
this.inRollBack = isInRollBackMode(NameNode.getStartupOption(conf)); this.inRollBack = isInRollBackMode(NameNode.getStartupOption(conf));

5
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
View File

@ -847,8 +847,9 @@
<name>dfs.namenode.safemode.extension</name> <name>dfs.namenode.safemode.extension</name>
<value>30000</value> <value>30000</value>
<description> <description>
Determines extension of safe mode in milliseconds Determines extension of safe mode in milliseconds after the threshold level
after the threshold level is reached. is reached. Support multiple time unit suffix (case insensitive), as
described in dfs.heartbeat.interval.
</description> </description>
</property> </property>

20
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManagerSafeMode.java
View File

@ -38,6 +38,7 @@
import java.io.InterruptedIOException; import java.io.InterruptedIOException;
import java.util.concurrent.TimeoutException; import java.util.concurrent.TimeoutException;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_DEFAULT;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
@ -392,6 +393,25 @@ public void testStayInSafeModeWhenBytesInFuture() throws Exception {
"are cleared.", bmSafeMode.leaveSafeMode(false)); "are cleared.", bmSafeMode.leaveSafeMode(false));
} }
@Test(timeout = 10000)
public void testExtensionConfig() {
final Configuration conf = new HdfsConfiguration();
bmSafeMode = new BlockManagerSafeMode(bm, fsn, false, conf);
assertEquals(DFS_NAMENODE_SAFEMODE_EXTENSION_DEFAULT, bmSafeMode.extension);
conf.set(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, "30000");
bmSafeMode = new BlockManagerSafeMode(bm, fsn, false, conf);
assertEquals(30000, bmSafeMode.extension);
conf.set(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, "20s");
bmSafeMode = new BlockManagerSafeMode(bm, fsn, false, conf);
assertEquals(20 * 1000, bmSafeMode.extension);
conf.set(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, "7m");
bmSafeMode = new BlockManagerSafeMode(bm, fsn, false, conf);
assertEquals(7 * 60 * 1000, bmSafeMode.extension);
}
/** /**
* Test get safe mode tip. * Test get safe mode tip.
*/ */