From 2d6049f1d683529c9e6372f33ee9b91fcfdbd3bd Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Tue, 16 Jul 2013 14:04:33 +0000 Subject: [PATCH] HDFS-4992. Make balancer's mover thread count and dispatcher thread count configurable. Contributed by Max Lapan git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1503720 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 ++++ .../hadoop/hdfs/server/balancer/Balancer.java | 17 ++++++++++------- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index b204a95657..76a5514b5f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -443,6 +443,9 @@ Release 2.1.0-beta - 2013-07-02 HDFS-4903. Print trash configuration and trash emptier state in namenode log. (Arpit Agarwal via suresh) + HDFS-4992. Make balancer's mover thread count and dispatcher thread count + configurable. (Max Lapan via szetszwo) + OPTIMIZATIONS HDFS-4465. Optimize datanode ReplicasMap and ReplicaInfo. (atm) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index b39e786ab9..dd698d51a8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -276,6 +276,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final boolean DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL_DEFAULT = false; public static final String DFS_BALANCER_MOVEDWINWIDTH_KEY = "dfs.balancer.movedWinWidth"; public static final long DFS_BALANCER_MOVEDWINWIDTH_DEFAULT = 5400*1000L; + public static final String DFS_BALANCER_MOVERTHREADS_KEY = "dfs.balancer.moverThreads"; + public static final int DFS_BALANCER_MOVERTHREADS_DEFAULT = 1000; + public static final String DFS_BALANCER_DISPATCHERTHREADS_KEY = "dfs.balancer.dispatcherThreads"; + public static final int DFS_BALANCER_DISPATCHERTHREADS_DEFAULT = 200; public static final String DFS_DATANODE_ADDRESS_KEY = "dfs.datanode.address"; public static final int DFS_DATANODE_DEFAULT_PORT = 50010; public static final String DFS_DATANODE_ADDRESS_DEFAULT = "0.0.0.0:" + DFS_DATANODE_DEFAULT_PORT; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java index 109ba71973..1ed0e1915f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java @@ -226,13 +226,9 @@ public class Balancer { = new HashMap(); private NetworkTopology cluster; - final static private int MOVER_THREAD_POOL_SIZE = 1000; - final private ExecutorService moverExecutor = - Executors.newFixedThreadPool(MOVER_THREAD_POOL_SIZE); - final static private int DISPATCHER_THREAD_POOL_SIZE = 200; - final private ExecutorService dispatcherExecutor = - Executors.newFixedThreadPool(DISPATCHER_THREAD_POOL_SIZE); - + + private final ExecutorService moverExecutor; + private final ExecutorService dispatcherExecutor; /* This class keeps track of a scheduled block move */ private class PendingBlockMove { @@ -830,6 +826,13 @@ private static void checkReplicationPolicyCompatibility(Configuration conf this.policy = p.policy; this.nnc = theblockpool; cluster = NetworkTopology.getInstance(conf); + + this.moverExecutor = Executors.newFixedThreadPool( + conf.getInt(DFSConfigKeys.DFS_BALANCER_MOVERTHREADS_KEY, + DFSConfigKeys.DFS_BALANCER_MOVERTHREADS_DEFAULT)); + this.dispatcherExecutor = Executors.newFixedThreadPool( + conf.getInt(DFSConfigKeys.DFS_BALANCER_DISPATCHERTHREADS_KEY, + DFSConfigKeys.DFS_BALANCER_DISPATCHERTHREADS_DEFAULT)); } /* Shuffle datanode array */