HADOOP-18373. IOStatisticsContext tuning (#4705)

The name of the option to enable/disable thread level statistics is
"fs.iostatistics.thread.level.enabled";

There is also an enabled() probe in IOStatisticsContext which can
be used to see if the thread level statistics is active.

Contributed by Viraj Jasani
This commit is contained in:
Viraj Jasani 2022-08-08 02:42:57 -07:00 committed by GitHub
parent d8d3325d2f
commit 06f0f7db79
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 34 additions and 7 deletions

View File

@ -480,13 +480,13 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
* Thread-level IOStats Support. * Thread-level IOStats Support.
* {@value} * {@value}
*/ */
public static final String THREAD_LEVEL_IOSTATISTICS_ENABLED = public static final String IOSTATISTICS_THREAD_LEVEL_ENABLED =
"fs.thread.level.iostatistics.enabled"; "fs.iostatistics.thread.level.enabled";
/** /**
* Default value for Thread-level IOStats Support is true. * Default value for Thread-level IOStats Support is true.
*/ */
public static final boolean THREAD_LEVEL_IOSTATISTICS_ENABLED_DEFAULT = public static final boolean IOSTATISTICS_THREAD_LEVEL_ENABLED_DEFAULT =
true; true;
} }

View File

@ -80,4 +80,14 @@ static void setThreadIOStatisticsContext(
IOStatisticsContextIntegration.setThreadIOStatisticsContext( IOStatisticsContextIntegration.setThreadIOStatisticsContext(
statisticsContext); statisticsContext);
} }
/**
* Static probe to check if the thread-level IO statistics enabled.
*
* @return if the thread-level IO statistics enabled.
*/
static boolean enabled() {
return IOStatisticsContextIntegration.isIOStatisticsThreadLevelEnabled();
}
} }

View File

@ -29,8 +29,8 @@
import org.apache.hadoop.fs.impl.WeakReferenceThreadMap; import org.apache.hadoop.fs.impl.WeakReferenceThreadMap;
import org.apache.hadoop.fs.statistics.IOStatisticsContext; import org.apache.hadoop.fs.statistics.IOStatisticsContext;
import static org.apache.hadoop.fs.CommonConfigurationKeys.THREAD_LEVEL_IOSTATISTICS_ENABLED; import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_THREAD_LEVEL_ENABLED;
import static org.apache.hadoop.fs.CommonConfigurationKeys.THREAD_LEVEL_IOSTATISTICS_ENABLED_DEFAULT; import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_THREAD_LEVEL_ENABLED_DEFAULT;
/** /**
* A Utility class for IOStatisticsContext, which helps in creating and * A Utility class for IOStatisticsContext, which helps in creating and
@ -76,8 +76,17 @@ public final class IOStatisticsContextIntegration {
// Work out if the current context has thread level IOStatistics enabled. // Work out if the current context has thread level IOStatistics enabled.
final Configuration configuration = new Configuration(); final Configuration configuration = new Configuration();
isThreadIOStatsEnabled = isThreadIOStatsEnabled =
configuration.getBoolean(THREAD_LEVEL_IOSTATISTICS_ENABLED, configuration.getBoolean(IOSTATISTICS_THREAD_LEVEL_ENABLED,
THREAD_LEVEL_IOSTATISTICS_ENABLED_DEFAULT); IOSTATISTICS_THREAD_LEVEL_ENABLED_DEFAULT);
}
/**
* Static probe to check if the thread-level IO statistics enabled.
*
* @return if the thread-level IO statistics enabled.
*/
public static boolean isIOStatisticsThreadLevelEnabled() {
return isThreadIOStatsEnabled;
} }
/** /**

View File

@ -153,6 +153,8 @@ public void testS3AInputStreamIOStatisticsContext()
* @return thread context * @return thread context
*/ */
private static IOStatisticsContext getAndResetThreadStatisticsContext() { private static IOStatisticsContext getAndResetThreadStatisticsContext() {
assertTrue("thread-level IOStatistics should be enabled by default",
IOStatisticsContext.enabled());
IOStatisticsContext context = IOStatisticsContext context =
IOStatisticsContext.getCurrentIOStatisticsContext(); IOStatisticsContext.getCurrentIOStatisticsContext();
context.reset(); context.reset();

View File

@ -178,6 +178,12 @@
<value>true</value> <value>true</value>
</property> </property>
<!-- Enable IOStatisticsContext support for Thread level. -->
<property>
<name>fs.iostatistics.thread.level.enabled</name>
<value>true</value>
</property>
<!-- <!--
To run these tests. To run these tests.