From 9a65a9aadce101300f1559388b15f9b0eefa8e25 Mon Sep 17 00:00:00 2001 From: Colin McCabe Date: Sat, 15 Mar 2014 07:52:19 +0000 Subject: [PATCH] HDFS-6106. Reduce default for dfs.namenode.path.based.cache.refresh.interval.ms (cmccabe) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1577798 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 ++-- .../hadoop-hdfs/src/main/resources/hdfs-default.xml | 6 +++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 7cc7668c31..4faa32ca8d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -406,6 +406,9 @@ Release 2.4.0 - UNRELEASED HDFS-6084. Namenode UI - "Hadoop" logo link shouldn't go to hadoop homepage. (Travis Thompson via wheat9) + HDFS-6106. Reduce default for + dfs.namenode.path.based.cache.refresh.interval.ms (cmccabe) + OPTIMIZATIONS HDFS-5790. LeaseManager.findPath is very slow when many leases need recovery diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index dd15ae8385..11c400e608 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -242,7 +242,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final int DFS_NAMENODE_LIST_CACHE_DIRECTIVES_NUM_RESPONSES_DEFAULT = 100; public static final String DFS_NAMENODE_PATH_BASED_CACHE_REFRESH_INTERVAL_MS = "dfs.namenode.path.based.cache.refresh.interval.ms"; - public static final long DFS_NAMENODE_PATH_BASED_CACHE_REFRESH_INTERVAL_MS_DEFAULT = 300000L; + public static final long DFS_NAMENODE_PATH_BASED_CACHE_REFRESH_INTERVAL_MS_DEFAULT = 30000L; // Whether to enable datanode's stale state detection and usage for reads public static final String DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_KEY = "dfs.namenode.avoid.read.stale.datanode"; @@ -370,7 +370,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_HEARTBEAT_INTERVAL_KEY = "dfs.heartbeat.interval"; public static final long DFS_HEARTBEAT_INTERVAL_DEFAULT = 3; public static final String DFS_NAMENODE_PATH_BASED_CACHE_RETRY_INTERVAL_MS = "dfs.namenode.path.based.cache.retry.interval.ms"; - public static final long DFS_NAMENODE_PATH_BASED_CACHE_RETRY_INTERVAL_MS_DEFAULT = 60000L; + public static final long DFS_NAMENODE_PATH_BASED_CACHE_RETRY_INTERVAL_MS_DEFAULT = 30000L; public static final String DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY = "dfs.namenode.decommission.interval"; public static final int DFS_NAMENODE_DECOMMISSION_INTERVAL_DEFAULT = 30; public static final String DFS_NAMENODE_DECOMMISSION_NODES_PER_INTERVAL_KEY = "dfs.namenode.decommission.nodes.per.interval"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 62236ed9f4..b43e8b7119 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -1662,19 +1662,19 @@ dfs.namenode.path.based.cache.refresh.interval.ms - 300000 + 30000 The amount of milliseconds between subsequent path cache rescans. Path cache rescans are when we calculate which blocks should be cached, and on what datanodes. - By default, this parameter is set to 300000, which is five minutes. + By default, this parameter is set to 30 seconds. dfs.namenode.path.based.cache.retry.interval.ms - 60000 + 30000 When the NameNode needs to uncache something that is cached, or cache something that is not cached, it must direct the DataNodes to do so by