From bd46bdf9f9244f3f3474d316255ac98717ed5719 Mon Sep 17 00:00:00 2001 From: He Xiaoqiao Date: Mon, 10 Jun 2019 17:20:50 -0700 Subject: [PATCH] HDFS-14553. Make queue size of BlockReportProcessingThread configurable. Contributed by He Xiaoqiao. Signed-off-by: Wei-Chiu Chuang --- .../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 ++++ .../hdfs/server/blockmanagement/BlockManager.java | 14 +++++++++----- .../src/main/resources/hdfs-default.xml | 8 ++++++++ 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index e9d0eec8d2..7bb30a6494 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -269,6 +269,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys { = "dfs.namenode.storageinfo.defragment.ratio"; public static final double DFS_NAMENODE_STORAGEINFO_DEFRAGMENT_RATIO_DEFAULT = 0.75; + public static final String DFS_NAMENODE_BLOCKREPORT_QUEUE_SIZE_KEY + = "dfs.namenode.blockreport.queue.size"; + public static final int DFS_NAMENODE_BLOCKREPORT_QUEUE_SIZE_DEFAULT + = 1024; public static final String DFS_WEBHDFS_AUTHENTICATION_FILTER_KEY = "dfs.web.authentication.filter"; /* Phrased as below to avoid javac inlining as a constant, to match the behavior when this was AuthFilter.class.getName(). Note that if you change the import for AuthFilter, you diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 9cfa18098e..bc2141d986 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -317,8 +317,7 @@ public long getTotalECBlockGroups() { new Daemon(new StorageInfoDefragmenter()); /** Block report thread for handling async reports. */ - private final BlockReportProcessingThread blockReportThread = - new BlockReportProcessingThread(); + private final BlockReportProcessingThread blockReportThread; /** * Store blocks {@literal ->} datanodedescriptor(s) map of corrupt replicas. @@ -574,6 +573,11 @@ public BlockManager(final Namesystem namesystem, boolean haEnabled, bmSafeMode = new BlockManagerSafeMode(this, namesystem, haEnabled, conf); + int queueSize = conf.getInt( + DFSConfigKeys.DFS_NAMENODE_BLOCKREPORT_QUEUE_SIZE_KEY, + DFSConfigKeys.DFS_NAMENODE_BLOCKREPORT_QUEUE_SIZE_DEFAULT); + blockReportThread = new BlockReportProcessingThread(queueSize); + LOG.info("defaultReplication = {}", defaultReplication); LOG.info("maxReplication = {}", maxReplication); LOG.info("minReplication = {}", minReplication); @@ -4966,11 +4970,11 @@ private class BlockReportProcessingThread extends Thread { private static final long MAX_LOCK_HOLD_MS = 4; private long lastFull = 0; - private final BlockingQueue queue = - new ArrayBlockingQueue(1024); + private final BlockingQueue queue; - BlockReportProcessingThread() { + BlockReportProcessingThread(int size) { super("Block report processor"); + queue = new ArrayBlockingQueue<>(size); setDaemon(true); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 8c575af90c..76c0660ebd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -5337,4 +5337,12 @@ inter-DN QOP. + + + dfs.namenode.blockreport.queue.size + 1024 + + The queue size of BlockReportProcessingThread in BlockManager. + +