big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-11286. Make AsyncDispatcher#printEventDetailsExecutor thread pool parameter configurable. (#4824). Contributed by fanshilun.

Browse Source 
Signed-off-by: Ayush Saxena <ayushsaxena@apache.org>
This commit is contained in:
slfan1989 2022-09-11 01:30:50 +08:00 committed by GitHub
parent b2760520c3
commit cdcb448b78
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 94 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -2987,6 +2987,51 @@ public static boolean isAclEnabled(Configuration conf) {
public static final int
DEFAULT_YARN_DISPATCHER_CPU_MONITOR_SAMPLES_PER_MIN = 60;
/**
* Resource manager dispatcher has a thread pool that prints EventQueue,
* configure the corePoolSize of this thread pool,
* the meaning of corePoolSize is the number of threads to keep in the pool.
*/
public static final String YARN_DISPATCHER_PRINT_THREAD_POOL_CORE_POOL_SIZE =
YARN_PREFIX + "dispatcher.print-thread-pool.core-pool-size";
/**
* The minimum number of core threads for the
* Resource manager dispatcher is 1.
*/
public static final int DEFAULT_YARN_DISPATCHER_PRINT_THREAD_POOL_CORE_POOL_SIZE = 1;
/**
* Resource manager dispatcher has a thread pool that prints EventQueue,
* configure the maximumPoolSize of this thread pool,
* the meaning of maximumPoolSize is the maximum number of threads to allow in the pool.
*/
public static final String YARN_DISPATCHER_PRINT_THREAD_POOL_MAXIMUM_POOL_SIZE =
YARN_PREFIX + "dispatcher.print-thread-pool.maximum-pool-size";
/**
* The maximum number of core threads for the
* Resource manager dispatcher is 5.
*/
public static final int DEFAULT_YARN_DISPATCHER_PRINT_THREAD_POOL_MAXIMUM_POOL_SIZE = 5;
/**
* Resource manager dispatcher has a thread pool that prints EventQueue,
* configure the keepAliveTime of this thread pool,
* The meaning of keepAliveTime is as follows
* when the number of threads is greater than the core,
* this is the maximum time that excess idle threads will wait for new tasks before terminating.
*/
public static final String YARN_DISPATCHER_PRINT_THREAD_POOL_KEEP_ALIVE_TIME =
YARN_PREFIX + "dispatcher.print-thread-pool.keep-alive-time";
/**
* The keep alive time of core threads for the
* Resource manager dispatcher is 10s.
*/
public static final long DEFAULT_YARN_DISPATCHER_PRINT_THREAD_POOL_KEEP_ALIVE_TIME =
10*1000; // 10s
/**
* CLASSPATH for YARN applications. A comma-separated list of CLASSPATH
* entries

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
View File

@ -191,8 +191,21 @@ protected void serviceInit(Configuration conf) throws Exception{
.build();
// Thread pool for async print event details,
// to prevent wasting too much time for RM.
int numCorePoolSizeThreads = getConfig().getInt(
YarnConfiguration.YARN_DISPATCHER_PRINT_THREAD_POOL_CORE_POOL_SIZE,
YarnConfiguration.DEFAULT_YARN_DISPATCHER_PRINT_THREAD_POOL_CORE_POOL_SIZE);
int numMaximumPoolSizeThreads = getConfig().getInt(
YarnConfiguration.YARN_DISPATCHER_PRINT_THREAD_POOL_MAXIMUM_POOL_SIZE,
YarnConfiguration.DEFAULT_YARN_DISPATCHER_PRINT_THREAD_POOL_MAXIMUM_POOL_SIZE);
long keepAliveTime =
conf.getTimeDuration(YarnConfiguration.YARN_DISPATCHER_PRINT_THREAD_POOL_KEEP_ALIVE_TIME,
YarnConfiguration.DEFAULT_YARN_DISPATCHER_PRINT_THREAD_POOL_KEEP_ALIVE_TIME,
TimeUnit.SECONDS);
printEventDetailsExecutor = new ThreadPoolExecutor(
1, 5, 10, TimeUnit.SECONDS,
numCorePoolSizeThreads, numMaximumPoolSizeThreads, keepAliveTime, TimeUnit.SECONDS,
new LinkedBlockingQueue<>(), threadFactory);
}

35
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -133,6 +133,41 @@
<value>60</value>
</property>
<property>
<description>
Resource manager dispatcher has a thread pool that prints EventQueue,
configure the corePoolSize of this thread pool,
the meaning of corePoolSize is the number of threads to keep in the pool.
the default value is 1.
</description>
<name>yarn.dispatcher.print-thread-pool.core-pool-size</name>
<value>1</value>
</property>
<property>
<description>
Resource manager dispatcher has a thread pool that prints EventQueue,
configure the maximumPoolSize of this thread pool,
the meaning of maximumPoolSize is the maximum number of threads to allow in the pool.
the default value is 5.
</description>
<name>yarn.dispatcher.print-thread-pool.maximum-pool-size</name>
<value>5</value>
</property>
<property>
<description>
Resource manager dispatcher has a thread pool that prints EventQueue,
configure the keepAliveTime of this thread pool,
The meaning of keepAliveTime is as follows
when the number of threads is greater than the core,
this is the maximum time that excess idle threads will wait for new tasks before terminating.
the default value is 10s.
</description>
<name>yarn.dispatcher.print-thread-pool.keep-alive-time</name>
<value>10s</value>
</property>
<property>
<description>The expiry interval for application master reporting.</description>
<name>yarn.am.liveness-monitor.expiry-interval-ms</name>