From 54f738575c0f0e90aec06670af05cd226997eaae Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Thu, 26 Jan 2012 21:37:28 +0000 Subject: [PATCH] MAPREDUCE-3699. Increased RPC handlers for all YARN servers to reasonable values for working at scale. Contributed by Hitesh Shah. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1236396 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../org/apache/hadoop/mapreduce/MRJobConfig.java | 2 +- .../src/main/resources/mapred-default.xml | 14 ++++++++++++++ .../apache/hadoop/yarn/conf/YarnConfiguration.java | 8 ++++---- .../src/main/resources/yarn-default.xml | 8 ++++---- 5 files changed, 26 insertions(+), 9 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 56cfd1335f..aa14dfbad4 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -200,6 +200,9 @@ Release 0.23.1 - Unreleased MAPREDUCE-3737. The Web Application Proxy's is not documented very well. (Robert Evans via mahadev) + MAPREDUCE-3699. Increased RPC handlers for all YARN servers to reasonable + values for working at scale. (Hitesh Shah via vinodkv) + OPTIMIZATIONS MAPREDUCE-3567. Extraneous JobConf objects in AM heap. (Vinod Kumar diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index a1c4407f26..7b684c5b61 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -412,7 +412,7 @@ public interface MRJobConfig { /** The number of threads used to handle task RPC calls.*/ public static final String MR_AM_TASK_LISTENER_THREAD_COUNT = MR_AM_PREFIX + "job.task.listener.thread-count"; - public static final int DEFAULT_MR_AM_TASK_LISTENER_THREAD_COUNT = 10; + public static final int DEFAULT_MR_AM_TASK_LISTENER_THREAD_COUNT = 30; /** How often the AM should send heartbeats to the RM.*/ public static final String MR_AM_TO_RM_HEARTBEAT_INTERVAL_MS = diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 5d02250c49..fc5b8b23f8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -1224,4 +1224,18 @@ mapreduce.job.end-notification.max.retry.interval + + yarn.app.mapreduce.am.job.task.listener.thread-count + 30 + The number of threads used to handle RPC calls in the + MR AppMaster from remote tasks + + + + yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms + 1000 + The interval in ms at which the MR AppMaster should send + heartbeats to the ResourceManager + + diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index a030079779..f4cbf6e65b 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -90,7 +90,7 @@ public class YarnConfiguration extends Configuration { /** The number of threads used to handle applications manager requests.*/ public static final String RM_CLIENT_THREAD_COUNT = RM_PREFIX + "client.thread-count"; - public static final int DEFAULT_RM_CLIENT_THREAD_COUNT = 10; + public static final int DEFAULT_RM_CLIENT_THREAD_COUNT = 50; /** The Kerberos principal for the resource manager.*/ public static final String RM_PRINCIPAL = @@ -106,7 +106,7 @@ public class YarnConfiguration extends Configuration { /** Number of threads to handle scheduler interface.*/ public static final String RM_SCHEDULER_CLIENT_THREAD_COUNT = RM_PREFIX + "scheduler.client.thread-count"; - public static final int DEFAULT_RM_SCHEDULER_CLIENT_THREAD_COUNT = 10; + public static final int DEFAULT_RM_SCHEDULER_CLIENT_THREAD_COUNT = 50; /** The address of the RM web application.*/ public static final String RM_WEBAPP_ADDRESS = @@ -184,7 +184,7 @@ public class YarnConfiguration extends Configuration { /** Number of threads to handle resource tracker calls.*/ public static final String RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT = RM_PREFIX + "resource-tracker.client.thread-count"; - public static final int DEFAULT_RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT = 10; + public static final int DEFAULT_RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT = 50; /** The class to use as the resource scheduler.*/ public static final String RM_SCHEDULER = @@ -257,7 +257,7 @@ public class YarnConfiguration extends Configuration { /** Number of threads container manager uses.*/ public static final String NM_CONTAINER_MGR_THREAD_COUNT = NM_PREFIX + "container-manager.thread-count"; - public static final int DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT = 5; + public static final int DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT = 20; /** Number of threads used in cleanup.*/ public static final String NM_DELETE_THREAD_COUNT = diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/resources/yarn-default.xml b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/resources/yarn-default.xml index b9e5ea47e4..cea45798a7 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/resources/yarn-default.xml +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/resources/yarn-default.xml @@ -67,7 +67,7 @@ The number of threads used to handle applications manager requests. yarn.resourcemanager.client.thread-count - 10 + 50 @@ -90,7 +90,7 @@ Number of threads to handle scheduler interface. yarn.resourcemanager.scheduler.client.thread-count - 10 + 50 @@ -179,7 +179,7 @@ Number of threads to handle resource tracker calls. yarn.resourcemanager.resource-tracker.client.thread-count - 10 + 50 @@ -244,7 +244,7 @@ Number of threads container manager uses. yarn.nodemanager.container-manager.thread-count - 5 + 20