diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index f0f8f3e151..459b9f8e35 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -290,6 +290,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys { = "dfs.namenode.blockreport.queue.size"; public static final int DFS_NAMENODE_BLOCKREPORT_QUEUE_SIZE_DEFAULT = 1024; + public static final String DFS_NAMENODE_BLOCKREPORT_MAX_LOCK_HOLD_TIME + = "dfs.namenode.blockreport.max.lock.hold.time"; + public static final long + DFS_NAMENODE_BLOCKREPORT_MAX_LOCK_HOLD_TIME_DEFAULT = 4; @Deprecated public static final String DFS_WEBHDFS_USER_PATTERN_KEY = HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 45d3b79249..aaa4aeb538 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -393,6 +393,9 @@ public long getTotalECBlockGroups() { // Max number of blocks to log info about during a block report. private final long maxNumBlocksToLog; + // Max write lock hold time for BlockReportProcessingThread(ms). + private final long maxLockHoldTime; + /** * When running inside a Standby node, the node may receive block reports * from datanodes before receiving the corresponding namespace edits from @@ -543,6 +546,10 @@ public BlockManager(final Namesystem namesystem, boolean haEnabled, this.maxNumBlocksToLog = conf.getLong(DFSConfigKeys.DFS_MAX_NUM_BLOCKS_TO_LOG_KEY, DFSConfigKeys.DFS_MAX_NUM_BLOCKS_TO_LOG_DEFAULT); + this.maxLockHoldTime = conf.getTimeDuration( + DFSConfigKeys.DFS_NAMENODE_BLOCKREPORT_MAX_LOCK_HOLD_TIME, + DFSConfigKeys.DFS_NAMENODE_BLOCKREPORT_MAX_LOCK_HOLD_TIME_DEFAULT, + TimeUnit.MILLISECONDS); this.numBlocksPerIteration = conf.getInt( DFSConfigKeys.DFS_BLOCK_MISREPLICATION_PROCESSING_LIMIT, DFSConfigKeys.DFS_BLOCK_MISREPLICATION_PROCESSING_LIMIT_DEFAULT); @@ -5159,7 +5166,6 @@ public int getBlockOpQueueLength() { } private class BlockReportProcessingThread extends Thread { - private static final long MAX_LOCK_HOLD_MS = 4; private long lastFull = 0; private final BlockingQueue queue; @@ -5195,7 +5201,7 @@ private void processQueue() { do { processed++; action.run(); - if (Time.monotonicNow() - start > MAX_LOCK_HOLD_MS) { + if (Time.monotonicNow() - start > maxLockHoldTime) { break; } action = queue.poll(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 5daaffbe3a..abb8dec925 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -5736,6 +5736,14 @@ + + dfs.namenode.blockreport.max.lock.hold.time + 4 + + The BlockReportProcessingThread max write lock hold time in ms. + + + dfs.journalnode.edits.dir.perm 700