From 3ddb52cfc0297632b917774c8bc11147b20b7455 Mon Sep 17 00:00:00 2001 From: Arun Murthy Date: Mon, 10 Oct 2011 03:06:47 +0000 Subject: [PATCH] MAPREDUCE-2876. Use a different config for ContainerAllocationExpirer. Contributed by Anupam Seth. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1180767 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../hadoop/yarn/conf/YarnConfiguration.java | 19 ++++--------------- .../resourcemanager/NMLivelinessMonitor.java | 9 ++++----- .../rmapp/attempt/AMLivelinessMonitor.java | 8 ++++---- .../ContainerAllocationExpirer.java | 10 +++++----- 5 files changed, 20 insertions(+), 29 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 5fc06c3d4c..1f878ecc51 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -1546,6 +1546,9 @@ Release 0.23.0 - Unreleased MAPREDUCE-2802. Ensure JobHistory filenames have jobId. (Jonathan Eagles via acmurthy) + MAPREDUCE-2876. Use a different config for ContainerAllocationExpirer. + (Anupam Seth via acmurthy) + Release 0.22.0 - Unreleased INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 780561d81a..23be2a0d50 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -146,22 +146,11 @@ public class YarnConfiguration extends Configuration { RM_PREFIX + "admin.client.thread-count"; public static final int DEFAULT_RM_ADMIN_CLIENT_THREAD_COUNT = 1; - /** How often should the RM check that the AM is still alive.*/ - public static final String RM_AM_LIVENESS_MONITOR_INTERVAL_MS = - RM_PREFIX + "amliveliness-monitor.interval-ms"; - public static final int DEFAULT_RM_AM_LIVENESS_MONITOR_INTERVAL_MS = 1000; - /** The maximum number of application master retries.*/ public static final String RM_AM_MAX_RETRIES = RM_PREFIX + "am.max-retries"; public static final int DEFAULT_RM_AM_MAX_RETRIES = 1; - /** How often to check that containers are still alive. */ - public static final String RM_CONTAINER_LIVENESS_MONITOR_INTERVAL_MS = - RM_PREFIX + "container.liveness-monitor.interval-ms"; - public static final int DEFAULT_RM_CONTAINER_LIVENESS_MONITOR_INTERVAL_MS = - 600000; - /** The keytab for the resource manager.*/ public static final String RM_KEYTAB = RM_PREFIX + "keytab"; @@ -171,10 +160,10 @@ public class YarnConfiguration extends Configuration { RM_PREFIX + "nm.liveness-monitor.expiry-interval-ms"; public static final int DEFAULT_RM_NM_EXPIRY_INTERVAL_MS = 600000; - /** How often to check that node managers are still alive.*/ - public static final String RM_NM_LIVENESS_MONITOR_INTERVAL_MS = - RM_PREFIX + "nm.liveness-monitor.interval-ms"; - public static final int DEFAULT_RM_NM_LIVENESS_MONITOR_INTERVAL_MS = 1000; + /** How long to wait until a container is considered dead.*/ + public static final String RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS = + RM_PREFIX + "rm.container-allocation.expiry-interval-ms"; + public static final int DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS = 600000; /** Path to file with nodes to include.*/ public static final String RM_NODES_INCLUDE_FILE_PATH = diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NMLivelinessMonitor.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NMLivelinessMonitor.java index ab89fbcbcc..7f901c4952 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NMLivelinessMonitor.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NMLivelinessMonitor.java @@ -39,11 +39,10 @@ public NMLivelinessMonitor(Dispatcher d) { public void init(Configuration conf) { super.init(conf); - setExpireInterval(conf.getInt(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, - YarnConfiguration.DEFAULT_RM_NM_EXPIRY_INTERVAL_MS)); - setMonitorInterval(conf.getInt( - YarnConfiguration.RM_NM_LIVENESS_MONITOR_INTERVAL_MS, - YarnConfiguration.DEFAULT_RM_NM_LIVENESS_MONITOR_INTERVAL_MS)); + int expireIntvl = conf.getInt(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, + YarnConfiguration.DEFAULT_RM_NM_EXPIRY_INTERVAL_MS); + setExpireInterval(expireIntvl); + setMonitorInterval(expireIntvl/3); } @Override diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AMLivelinessMonitor.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AMLivelinessMonitor.java index 2f8d82e9f7..7e52d5a49d 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AMLivelinessMonitor.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AMLivelinessMonitor.java @@ -37,10 +37,10 @@ public AMLivelinessMonitor(Dispatcher d) { public void init(Configuration conf) { super.init(conf); - setExpireInterval(conf.getInt(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, - YarnConfiguration.DEFAULT_RM_AM_EXPIRY_INTERVAL_MS)); - setMonitorInterval(conf.getInt(YarnConfiguration.RM_AM_LIVENESS_MONITOR_INTERVAL_MS, - YarnConfiguration.DEFAULT_RM_AM_LIVENESS_MONITOR_INTERVAL_MS)); + int expireIntvl = conf.getInt(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, + YarnConfiguration.DEFAULT_RM_AM_EXPIRY_INTERVAL_MS); + setExpireInterval(expireIntvl); + setMonitorInterval(expireIntvl/3); } @Override diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerAllocationExpirer.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerAllocationExpirer.java index 9216364ff5..0e65a9b66d 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerAllocationExpirer.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerAllocationExpirer.java @@ -39,11 +39,11 @@ public ContainerAllocationExpirer(Dispatcher d) { public void init(Configuration conf) { super.init(conf); - setExpireInterval(conf.getInt( - YarnConfiguration.RM_CONTAINER_LIVENESS_MONITOR_INTERVAL_MS, - YarnConfiguration.DEFAULT_RM_CONTAINER_LIVENESS_MONITOR_INTERVAL_MS)); - setMonitorInterval(conf.getInt(YarnConfiguration.RM_AM_LIVENESS_MONITOR_INTERVAL_MS, - YarnConfiguration.DEFAULT_RM_AM_LIVENESS_MONITOR_INTERVAL_MS)); + int expireIntvl = conf.getInt( + YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS, + YarnConfiguration.DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS); + setExpireInterval(expireIntvl); + setMonitorInterval(expireIntvl/3); } @Override