big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Projects Wiki Activity

HDFS-15041. Make MAX_LOCK_HOLD_MS and full queue size configurable. Contributed by zhuqi.

Browse Source
This commit is contained in:
Ayush Saxena 2020-02-23 14:25:39 +05:30
parent ba9025c7cd
commit 9eb7a8bdf8
3 changed files with 20 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
View File

@ -290,6 +290,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
= "dfs.namenode.blockreport.queue.size"; = "dfs.namenode.blockreport.queue.size";
public static final int DFS_NAMENODE_BLOCKREPORT_QUEUE_SIZE_DEFAULT public static final int DFS_NAMENODE_BLOCKREPORT_QUEUE_SIZE_DEFAULT
= 1024; = 1024;
public static final String DFS_NAMENODE_BLOCKREPORT_MAX_LOCK_HOLD_TIME
= "dfs.namenode.blockreport.max.lock.hold.time";
public static final long
DFS_NAMENODE_BLOCKREPORT_MAX_LOCK_HOLD_TIME_DEFAULT = 4;
@Deprecated @Deprecated
public static final String DFS_WEBHDFS_USER_PATTERN_KEY = public static final String DFS_WEBHDFS_USER_PATTERN_KEY =
HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY; HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY;

10
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
View File

@ -393,6 +393,9 @@ public class BlockManager implements BlockStatsMXBean {
// Max number of blocks to log info about during a block report. // Max number of blocks to log info about during a block report.
private final long maxNumBlocksToLog; private final long maxNumBlocksToLog;
// Max write lock hold time for BlockReportProcessingThread(ms).
private final long maxLockHoldTime;
/** /**
* When running inside a Standby node, the node may receive block reports * When running inside a Standby node, the node may receive block reports
* from datanodes before receiving the corresponding namespace edits from * from datanodes before receiving the corresponding namespace edits from
@ -543,6 +546,10 @@ public class BlockManager implements BlockStatsMXBean {
this.maxNumBlocksToLog = this.maxNumBlocksToLog =
conf.getLong(DFSConfigKeys.DFS_MAX_NUM_BLOCKS_TO_LOG_KEY, conf.getLong(DFSConfigKeys.DFS_MAX_NUM_BLOCKS_TO_LOG_KEY,
DFSConfigKeys.DFS_MAX_NUM_BLOCKS_TO_LOG_DEFAULT); DFSConfigKeys.DFS_MAX_NUM_BLOCKS_TO_LOG_DEFAULT);
this.maxLockHoldTime = conf.getTimeDuration(
DFSConfigKeys.DFS_NAMENODE_BLOCKREPORT_MAX_LOCK_HOLD_TIME,
DFSConfigKeys.DFS_NAMENODE_BLOCKREPORT_MAX_LOCK_HOLD_TIME_DEFAULT,
TimeUnit.MILLISECONDS);
this.numBlocksPerIteration = conf.getInt( this.numBlocksPerIteration = conf.getInt(
DFSConfigKeys.DFS_BLOCK_MISREPLICATION_PROCESSING_LIMIT, DFSConfigKeys.DFS_BLOCK_MISREPLICATION_PROCESSING_LIMIT,
DFSConfigKeys.DFS_BLOCK_MISREPLICATION_PROCESSING_LIMIT_DEFAULT); DFSConfigKeys.DFS_BLOCK_MISREPLICATION_PROCESSING_LIMIT_DEFAULT);
@ -5159,7 +5166,6 @@ public class BlockManager implements BlockStatsMXBean {
} }
private class BlockReportProcessingThread extends Thread { private class BlockReportProcessingThread extends Thread {
private static final long MAX_LOCK_HOLD_MS = 4;
private long lastFull = 0; private long lastFull = 0;
private final BlockingQueue<Runnable> queue; private final BlockingQueue<Runnable> queue;
@ -5195,7 +5201,7 @@ public class BlockManager implements BlockStatsMXBean {
do { do {
processed++; processed++;
action.run(); action.run();
if (Time.monotonicNow() - start > MAX_LOCK_HOLD_MS) { if (Time.monotonicNow() - start > maxLockHoldTime) {
break; break;
} }
action = queue.poll(); action = queue.poll();

8
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
View File

@ -5736,6 +5736,14 @@
</description> </description>
</property> </property>
<property>
<name>dfs.namenode.blockreport.max.lock.hold.time</name>
<value>4</value>
<description>
The BlockReportProcessingThread max write lock hold time in ms.
</description>
</property>
<property> <property>
<name>dfs.journalnode.edits.dir.perm</name> <name>dfs.journalnode.edits.dir.perm</name>
<value>700</value> <value>700</value>