From cdcb448b787378e752fec7dd8f3795e075c8f394 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Sun, 11 Sep 2022 01:30:50 +0800 Subject: [PATCH] YARN-11286. Make AsyncDispatcher#printEventDetailsExecutor thread pool parameter configurable. (#4824). Contributed by fanshilun. Signed-off-by: Ayush Saxena --- .../hadoop/yarn/conf/YarnConfiguration.java | 45 +++++++++++++++++++ .../hadoop/yarn/event/AsyncDispatcher.java | 15 ++++++- .../src/main/resources/yarn-default.xml | 35 +++++++++++++++ 3 files changed, 94 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index cc2c10cd2f..ad4bcf8495 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2987,6 +2987,51 @@ public static boolean isAclEnabled(Configuration conf) { public static final int DEFAULT_YARN_DISPATCHER_CPU_MONITOR_SAMPLES_PER_MIN = 60; + /** + * Resource manager dispatcher has a thread pool that prints EventQueue, + * configure the corePoolSize of this thread pool, + * the meaning of corePoolSize is the number of threads to keep in the pool. + */ + public static final String YARN_DISPATCHER_PRINT_THREAD_POOL_CORE_POOL_SIZE = + YARN_PREFIX + "dispatcher.print-thread-pool.core-pool-size"; + + /** + * The minimum number of core threads for the + * Resource manager dispatcher is 1. + */ + public static final int DEFAULT_YARN_DISPATCHER_PRINT_THREAD_POOL_CORE_POOL_SIZE = 1; + + /** + * Resource manager dispatcher has a thread pool that prints EventQueue, + * configure the maximumPoolSize of this thread pool, + * the meaning of maximumPoolSize is the maximum number of threads to allow in the pool. + */ + public static final String YARN_DISPATCHER_PRINT_THREAD_POOL_MAXIMUM_POOL_SIZE = + YARN_PREFIX + "dispatcher.print-thread-pool.maximum-pool-size"; + + /** + * The maximum number of core threads for the + * Resource manager dispatcher is 5. + */ + public static final int DEFAULT_YARN_DISPATCHER_PRINT_THREAD_POOL_MAXIMUM_POOL_SIZE = 5; + + /** + * Resource manager dispatcher has a thread pool that prints EventQueue, + * configure the keepAliveTime of this thread pool, + * The meaning of keepAliveTime is as follows + * when the number of threads is greater than the core, + * this is the maximum time that excess idle threads will wait for new tasks before terminating. + */ + public static final String YARN_DISPATCHER_PRINT_THREAD_POOL_KEEP_ALIVE_TIME = + YARN_PREFIX + "dispatcher.print-thread-pool.keep-alive-time"; + + /** + * The keep alive time of core threads for the + * Resource manager dispatcher is 10s. + */ + public static final long DEFAULT_YARN_DISPATCHER_PRINT_THREAD_POOL_KEEP_ALIVE_TIME = + 10*1000; // 10s + /** * CLASSPATH for YARN applications. A comma-separated list of CLASSPATH * entries diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java index 1c4ed24b47..eee927c8c7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java @@ -191,8 +191,21 @@ protected void serviceInit(Configuration conf) throws Exception{ .build(); // Thread pool for async print event details, // to prevent wasting too much time for RM. + int numCorePoolSizeThreads = getConfig().getInt( + YarnConfiguration.YARN_DISPATCHER_PRINT_THREAD_POOL_CORE_POOL_SIZE, + YarnConfiguration.DEFAULT_YARN_DISPATCHER_PRINT_THREAD_POOL_CORE_POOL_SIZE); + + int numMaximumPoolSizeThreads = getConfig().getInt( + YarnConfiguration.YARN_DISPATCHER_PRINT_THREAD_POOL_MAXIMUM_POOL_SIZE, + YarnConfiguration.DEFAULT_YARN_DISPATCHER_PRINT_THREAD_POOL_MAXIMUM_POOL_SIZE); + + long keepAliveTime = + conf.getTimeDuration(YarnConfiguration.YARN_DISPATCHER_PRINT_THREAD_POOL_KEEP_ALIVE_TIME, + YarnConfiguration.DEFAULT_YARN_DISPATCHER_PRINT_THREAD_POOL_KEEP_ALIVE_TIME, + TimeUnit.SECONDS); + printEventDetailsExecutor = new ThreadPoolExecutor( - 1, 5, 10, TimeUnit.SECONDS, + numCorePoolSizeThreads, numMaximumPoolSizeThreads, keepAliveTime, TimeUnit.SECONDS, new LinkedBlockingQueue<>(), threadFactory); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 9d95fd43c0..fb3af3fe61 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -133,6 +133,41 @@ 60 + + + Resource manager dispatcher has a thread pool that prints EventQueue, + configure the corePoolSize of this thread pool, + the meaning of corePoolSize is the number of threads to keep in the pool. + the default value is 1. + + yarn.dispatcher.print-thread-pool.core-pool-size + 1 + + + + + Resource manager dispatcher has a thread pool that prints EventQueue, + configure the maximumPoolSize of this thread pool, + the meaning of maximumPoolSize is the maximum number of threads to allow in the pool. + the default value is 5. + + yarn.dispatcher.print-thread-pool.maximum-pool-size + 5 + + + + + Resource manager dispatcher has a thread pool that prints EventQueue, + configure the keepAliveTime of this thread pool, + The meaning of keepAliveTime is as follows + when the number of threads is greater than the core, + this is the maximum time that excess idle threads will wait for new tasks before terminating. + the default value is 10s. + + yarn.dispatcher.print-thread-pool.keep-alive-time + 10s + + The expiry interval for application master reporting. yarn.am.liveness-monitor.expiry-interval-ms