From 06f0f7db7951b6d49da0270716263579ad317a81 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Mon, 8 Aug 2022 02:42:57 -0700 Subject: [PATCH] HADOOP-18373. IOStatisticsContext tuning (#4705) The name of the option to enable/disable thread level statistics is "fs.iostatistics.thread.level.enabled"; There is also an enabled() probe in IOStatisticsContext which can be used to see if the thread level statistics is active. Contributed by Viraj Jasani --- .../hadoop/fs/CommonConfigurationKeys.java | 6 +++--- .../fs/statistics/IOStatisticsContext.java | 10 ++++++++++ .../impl/IOStatisticsContextIntegration.java | 17 +++++++++++++---- .../fs/s3a/ITestS3AIOStatisticsContext.java | 2 ++ .../hadoop-aws/src/test/resources/core-site.xml | 6 ++++++ 5 files changed, 34 insertions(+), 7 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java index 7c54b32dc3..75b19e2ae2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java @@ -480,13 +480,13 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic { * Thread-level IOStats Support. * {@value} */ - public static final String THREAD_LEVEL_IOSTATISTICS_ENABLED = - "fs.thread.level.iostatistics.enabled"; + public static final String IOSTATISTICS_THREAD_LEVEL_ENABLED = + "fs.iostatistics.thread.level.enabled"; /** * Default value for Thread-level IOStats Support is true. */ - public static final boolean THREAD_LEVEL_IOSTATISTICS_ENABLED_DEFAULT = + public static final boolean IOSTATISTICS_THREAD_LEVEL_ENABLED_DEFAULT = true; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsContext.java index fb10b93848..557c57ea4d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsContext.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsContext.java @@ -80,4 +80,14 @@ static void setThreadIOStatisticsContext( IOStatisticsContextIntegration.setThreadIOStatisticsContext( statisticsContext); } + + /** + * Static probe to check if the thread-level IO statistics enabled. + * + * @return if the thread-level IO statistics enabled. + */ + static boolean enabled() { + return IOStatisticsContextIntegration.isIOStatisticsThreadLevelEnabled(); + } + } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsContextIntegration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsContextIntegration.java index 483d1e4570..2a394e6a1c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsContextIntegration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsContextIntegration.java @@ -29,8 +29,8 @@ import org.apache.hadoop.fs.impl.WeakReferenceThreadMap; import org.apache.hadoop.fs.statistics.IOStatisticsContext; -import static org.apache.hadoop.fs.CommonConfigurationKeys.THREAD_LEVEL_IOSTATISTICS_ENABLED; -import static org.apache.hadoop.fs.CommonConfigurationKeys.THREAD_LEVEL_IOSTATISTICS_ENABLED_DEFAULT; +import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_THREAD_LEVEL_ENABLED; +import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_THREAD_LEVEL_ENABLED_DEFAULT; /** * A Utility class for IOStatisticsContext, which helps in creating and @@ -76,8 +76,17 @@ public final class IOStatisticsContextIntegration { // Work out if the current context has thread level IOStatistics enabled. final Configuration configuration = new Configuration(); isThreadIOStatsEnabled = - configuration.getBoolean(THREAD_LEVEL_IOSTATISTICS_ENABLED, - THREAD_LEVEL_IOSTATISTICS_ENABLED_DEFAULT); + configuration.getBoolean(IOSTATISTICS_THREAD_LEVEL_ENABLED, + IOSTATISTICS_THREAD_LEVEL_ENABLED_DEFAULT); + } + + /** + * Static probe to check if the thread-level IO statistics enabled. + * + * @return if the thread-level IO statistics enabled. + */ + public static boolean isIOStatisticsThreadLevelEnabled() { + return isThreadIOStatsEnabled; } /** diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AIOStatisticsContext.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AIOStatisticsContext.java index 19c40c6466..d2ae9d7dc3 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AIOStatisticsContext.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AIOStatisticsContext.java @@ -153,6 +153,8 @@ public void testS3AInputStreamIOStatisticsContext() * @return thread context */ private static IOStatisticsContext getAndResetThreadStatisticsContext() { + assertTrue("thread-level IOStatistics should be enabled by default", + IOStatisticsContext.enabled()); IOStatisticsContext context = IOStatisticsContext.getCurrentIOStatisticsContext(); context.reset(); diff --git a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml index 1525f51d9d..600ea3a178 100644 --- a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml +++ b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml @@ -178,6 +178,12 @@ true + + + fs.iostatistics.thread.level.enabled + true + +