big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-18373. IOStatisticsContext tuning (#4705)

Browse Source 
The name of the option to enable/disable thread level statistics is
"fs.iostatistics.thread.level.enabled";

There is also an enabled() probe in IOStatisticsContext which can
be used to see if the thread level statistics is active.

Contributed by Viraj Jasani
This commit is contained in:
Viraj Jasani 2022-08-08 02:42:57 -07:00 committed by GitHub
parent d8d3325d2f
commit 06f0f7db79
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 34 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
View File

@ -480,13 +480,13 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
* Thread-level IOStats Support.
* {@value}
*/
public static final String THREAD_LEVEL_IOSTATISTICS_ENABLED =
"fs.thread.level.iostatistics.enabled";
public static final String IOSTATISTICS_THREAD_LEVEL_ENABLED =
"fs.iostatistics.thread.level.enabled";
/**
* Default value for Thread-level IOStats Support is true.
*/
public static final boolean THREAD_LEVEL_IOSTATISTICS_ENABLED_DEFAULT =
public static final boolean IOSTATISTICS_THREAD_LEVEL_ENABLED_DEFAULT =
true;
}

10
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsContext.java
View File

@ -80,4 +80,14 @@ static void setThreadIOStatisticsContext(
IOStatisticsContextIntegration.setThreadIOStatisticsContext(
statisticsContext);
}
/**
* Static probe to check if the thread-level IO statistics enabled.
*
* @return if the thread-level IO statistics enabled.
*/
static boolean enabled() {
return IOStatisticsContextIntegration.isIOStatisticsThreadLevelEnabled();
}
}

17
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsContextIntegration.java
View File

@ -29,8 +29,8 @@
import org.apache.hadoop.fs.impl.WeakReferenceThreadMap;
import org.apache.hadoop.fs.statistics.IOStatisticsContext;
import static org.apache.hadoop.fs.CommonConfigurationKeys.THREAD_LEVEL_IOSTATISTICS_ENABLED;
import static org.apache.hadoop.fs.CommonConfigurationKeys.THREAD_LEVEL_IOSTATISTICS_ENABLED_DEFAULT;
import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_THREAD_LEVEL_ENABLED;
import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_THREAD_LEVEL_ENABLED_DEFAULT;
/**
* A Utility class for IOStatisticsContext, which helps in creating and
@ -76,8 +76,17 @@ public final class IOStatisticsContextIntegration {
// Work out if the current context has thread level IOStatistics enabled.
final Configuration configuration = new Configuration();
isThreadIOStatsEnabled =
configuration.getBoolean(THREAD_LEVEL_IOSTATISTICS_ENABLED,
THREAD_LEVEL_IOSTATISTICS_ENABLED_DEFAULT);
configuration.getBoolean(IOSTATISTICS_THREAD_LEVEL_ENABLED,
IOSTATISTICS_THREAD_LEVEL_ENABLED_DEFAULT);
}
/**
* Static probe to check if the thread-level IO statistics enabled.
*
* @return if the thread-level IO statistics enabled.
*/
public static boolean isIOStatisticsThreadLevelEnabled() {
return isThreadIOStatsEnabled;
}
/**

2
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AIOStatisticsContext.java
View File

@ -153,6 +153,8 @@ public void testS3AInputStreamIOStatisticsContext()
* @return thread context
*/
private static IOStatisticsContext getAndResetThreadStatisticsContext() {
assertTrue("thread-level IOStatistics should be enabled by default",
IOStatisticsContext.enabled());
IOStatisticsContext context =
IOStatisticsContext.getCurrentIOStatisticsContext();
context.reset();

6
hadoop-tools/hadoop-aws/src/test/resources/core-site.xml
View File

@ -178,6 +178,12 @@
<value>true</value>
</property>
<!-- Enable IOStatisticsContext support for Thread level. -->
<property>
<name>fs.iostatistics.thread.level.enabled</name>
<value>true</value>
</property>
<!--
To run these tests.