big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-15041. Make MAX_LOCK_HOLD_MS and full queue size configurable. Contributed by zhuqi.

Browse Source
This commit is contained in:
Ayush Saxena 2020-02-23 14:25:39 +05:30
parent ba9025c7cd
commit 9eb7a8bdf8
3 changed files with 20 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
View File

@ -290,6 +290,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
= "dfs.namenode.blockreport.queue.size";
public static final int DFS_NAMENODE_BLOCKREPORT_QUEUE_SIZE_DEFAULT
= 1024;
public static final String DFS_NAMENODE_BLOCKREPORT_MAX_LOCK_HOLD_TIME
= "dfs.namenode.blockreport.max.lock.hold.time";
public static final long
DFS_NAMENODE_BLOCKREPORT_MAX_LOCK_HOLD_TIME_DEFAULT = 4;
@Deprecated
public static final String DFS_WEBHDFS_USER_PATTERN_KEY =
HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY;

10
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
View File

@ -393,6 +393,9 @@ public long getTotalECBlockGroups() {
// Max number of blocks to log info about during a block report.
private final long maxNumBlocksToLog;
// Max write lock hold time for BlockReportProcessingThread(ms).
private final long maxLockHoldTime;
/**
* When running inside a Standby node, the node may receive block reports
* from datanodes before receiving the corresponding namespace edits from
@ -543,6 +546,10 @@ public BlockManager(final Namesystem namesystem, boolean haEnabled,
this.maxNumBlocksToLog =
conf.getLong(DFSConfigKeys.DFS_MAX_NUM_BLOCKS_TO_LOG_KEY,
DFSConfigKeys.DFS_MAX_NUM_BLOCKS_TO_LOG_DEFAULT);
this.maxLockHoldTime = conf.getTimeDuration(
DFSConfigKeys.DFS_NAMENODE_BLOCKREPORT_MAX_LOCK_HOLD_TIME,
DFSConfigKeys.DFS_NAMENODE_BLOCKREPORT_MAX_LOCK_HOLD_TIME_DEFAULT,
TimeUnit.MILLISECONDS);
this.numBlocksPerIteration = conf.getInt(
DFSConfigKeys.DFS_BLOCK_MISREPLICATION_PROCESSING_LIMIT,
DFSConfigKeys.DFS_BLOCK_MISREPLICATION_PROCESSING_LIMIT_DEFAULT);
@ -5159,7 +5166,6 @@ public int getBlockOpQueueLength() {
}
private class BlockReportProcessingThread extends Thread {
private static final long MAX_LOCK_HOLD_MS = 4;
private long lastFull = 0;
private final BlockingQueue<Runnable> queue;
@ -5195,7 +5201,7 @@ private void processQueue() {
do {
processed++;
action.run();
if (Time.monotonicNow() - start > MAX_LOCK_HOLD_MS) {
if (Time.monotonicNow() - start > maxLockHoldTime) {
break;
}
action = queue.poll();

8
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
View File

@ -5736,6 +5736,14 @@
</description>
</property>
<property>
<name>dfs.namenode.blockreport.max.lock.hold.time</name>
<value>4</value>
<description>
The BlockReportProcessingThread max write lock hold time in ms.
</description>
</property>
<property>
<name>dfs.journalnode.edits.dir.perm</name>
<value>700</value>