big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-3699. Increased RPC handlers for all YARN servers to reasonable values for working at scale. Contributed by Hitesh Shah.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1236396 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2012-01-26 21:37:28 +00:00
parent af6448e543
commit 54f738575c
5 changed files with 26 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -200,6 +200,9 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3737. The Web Application Proxy's is not documented very well. MAPREDUCE-3737. The Web Application Proxy's is not documented very well.
(Robert Evans via mahadev) (Robert Evans via mahadev)
MAPREDUCE-3699. Increased RPC handlers for all YARN servers to reasonable
values for working at scale. (Hitesh Shah via vinodkv)
OPTIMIZATIONS OPTIMIZATIONS
MAPREDUCE-3567. Extraneous JobConf objects in AM heap. (Vinod Kumar MAPREDUCE-3567. Extraneous JobConf objects in AM heap. (Vinod Kumar

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
View File

@ -412,7 +412,7 @@ public interface MRJobConfig {
/** The number of threads used to handle task RPC calls.*/ /** The number of threads used to handle task RPC calls.*/
public static final String MR_AM_TASK_LISTENER_THREAD_COUNT = public static final String MR_AM_TASK_LISTENER_THREAD_COUNT =
MR_AM_PREFIX + "job.task.listener.thread-count"; MR_AM_PREFIX + "job.task.listener.thread-count";
public static final int DEFAULT_MR_AM_TASK_LISTENER_THREAD_COUNT = 10; public static final int DEFAULT_MR_AM_TASK_LISTENER_THREAD_COUNT = 30;
/** How often the AM should send heartbeats to the RM.*/ /** How often the AM should send heartbeats to the RM.*/
public static final String MR_AM_TO_RM_HEARTBEAT_INTERVAL_MS = public static final String MR_AM_TO_RM_HEARTBEAT_INTERVAL_MS =

14
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
View File

@ -1224,4 +1224,18 @@
mapreduce.job.end-notification.max.retry.interval</description> mapreduce.job.end-notification.max.retry.interval</description>
</property> </property>
<property>
<name>yarn.app.mapreduce.am.job.task.listener.thread-count</name>
<value>30</value>
<description>The number of threads used to handle RPC calls in the
MR AppMaster from remote tasks</description>
</property>
<property>
<name>yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms</name>
<value>1000</value>
<description>The interval in ms at which the MR AppMaster should send
heartbeats to the ResourceManager</description>
</property>
</configuration> </configuration>

8
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -90,7 +90,7 @@ public class YarnConfiguration extends Configuration {
/** The number of threads used to handle applications manager requests.*/ /** The number of threads used to handle applications manager requests.*/
public static final String RM_CLIENT_THREAD_COUNT = public static final String RM_CLIENT_THREAD_COUNT =
RM_PREFIX + "client.thread-count"; RM_PREFIX + "client.thread-count";
public static final int DEFAULT_RM_CLIENT_THREAD_COUNT = 10; public static final int DEFAULT_RM_CLIENT_THREAD_COUNT = 50;
/** The Kerberos principal for the resource manager.*/ /** The Kerberos principal for the resource manager.*/
public static final String RM_PRINCIPAL = public static final String RM_PRINCIPAL =
@ -106,7 +106,7 @@ public class YarnConfiguration extends Configuration {
/** Number of threads to handle scheduler interface.*/ /** Number of threads to handle scheduler interface.*/
public static final String RM_SCHEDULER_CLIENT_THREAD_COUNT = public static final String RM_SCHEDULER_CLIENT_THREAD_COUNT =
RM_PREFIX + "scheduler.client.thread-count"; RM_PREFIX + "scheduler.client.thread-count";
public static final int DEFAULT_RM_SCHEDULER_CLIENT_THREAD_COUNT = 10; public static final int DEFAULT_RM_SCHEDULER_CLIENT_THREAD_COUNT = 50;
/** The address of the RM web application.*/ /** The address of the RM web application.*/
public static final String RM_WEBAPP_ADDRESS = public static final String RM_WEBAPP_ADDRESS =
@ -184,7 +184,7 @@ public class YarnConfiguration extends Configuration {
/** Number of threads to handle resource tracker calls.*/ /** Number of threads to handle resource tracker calls.*/
public static final String RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT = public static final String RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT =
RM_PREFIX + "resource-tracker.client.thread-count"; RM_PREFIX + "resource-tracker.client.thread-count";
public static final int DEFAULT_RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT = 10; public static final int DEFAULT_RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT = 50;
/** The class to use as the resource scheduler.*/ /** The class to use as the resource scheduler.*/
public static final String RM_SCHEDULER = public static final String RM_SCHEDULER =
@ -257,7 +257,7 @@ public class YarnConfiguration extends Configuration {
/** Number of threads container manager uses.*/ /** Number of threads container manager uses.*/
public static final String NM_CONTAINER_MGR_THREAD_COUNT = public static final String NM_CONTAINER_MGR_THREAD_COUNT =
NM_PREFIX + "container-manager.thread-count"; NM_PREFIX + "container-manager.thread-count";
public static final int DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT = 5; public static final int DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT = 20;
/** Number of threads used in cleanup.*/ /** Number of threads used in cleanup.*/
public static final String NM_DELETE_THREAD_COUNT = public static final String NM_DELETE_THREAD_COUNT =

8
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/resources/yarn-default.xml
View File

@ -67,7 +67,7 @@
<property> <property>
<description>The number of threads used to handle applications manager requests.</description> <description>The number of threads used to handle applications manager requests.</description>
<name>yarn.resourcemanager.client.thread-count</name> <name>yarn.resourcemanager.client.thread-count</name>
<value>10</value> <value>50</value>
</property> </property>
<property> <property>
@ -90,7 +90,7 @@
<property> <property>
<description>Number of threads to handle scheduler interface.</description> <description>Number of threads to handle scheduler interface.</description>
<name>yarn.resourcemanager.scheduler.client.thread-count</name> <name>yarn.resourcemanager.scheduler.client.thread-count</name>
<value>10</value> <value>50</value>
</property> </property>
<property> <property>
@ -179,7 +179,7 @@
<property> <property>
<description>Number of threads to handle resource tracker calls.</description> <description>Number of threads to handle resource tracker calls.</description>
<name>yarn.resourcemanager.resource-tracker.client.thread-count</name> <name>yarn.resourcemanager.resource-tracker.client.thread-count</name>
<value>10</value> <value>50</value>
</property> </property>
<property> <property>
@ -244,7 +244,7 @@
<property> <property>
<description>Number of threads container manager uses.</description> <description>Number of threads container manager uses.</description>
<name>yarn.nodemanager.container-manager.thread-count</name> <name>yarn.nodemanager.container-manager.thread-count</name>
<value>5</value> <value>20</value>
</property> </property>
<property> <property>