big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-15814. Make some parameters configurable for DataNodeDiskMetrics for branch-3.3 (#3021)

Browse Source
This commit is contained in:
litao 2021-06-08 15:10:37 +08:00 committed by GitHub
parent 37516726d7
commit 46d4b51bff
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 46 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
View File

@ -655,6 +655,14 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final long
DFS_DATANODE_PEER_METRICS_MIN_OUTLIER_DETECTION_SAMPLES_DEFAULT =
1000;
public static final String DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY =
"dfs.datanode.min.outlier.detection.disks";
public static final long DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_DEFAULT =
5L;
public static final String DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY =
"dfs.datanode.slowdisk.low.threshold.ms";
public static final long DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_DEFAULT =
20L;
public static final String DFS_DATANODE_HOST_NAME_KEY =
HdfsClientConfigKeys.DeprecatedKeys.DFS_DATANODE_HOST_NAME_KEY;
public static final String DFS_NAMENODE_CHECKPOINT_DIR_KEY =

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
View File

@ -1458,7 +1458,7 @@ void startDataNode(List<StorageLocation> dataDirectories,
if (dnConf.diskStatsEnabled) {
diskMetrics = new DataNodeDiskMetrics(this,
dnConf.outliersReportIntervalMs);
dnConf.outliersReportIntervalMs, getConf());
}
}

26
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java
View File

@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hdfs.server.datanode.metrics;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
@ -48,8 +50,6 @@ public class DataNodeDiskMetrics {
DataNodeDiskMetrics.class);
private DataNode dn;
private final long MIN_OUTLIER_DETECTION_DISKS = 5;
private final long SLOW_DISK_LOW_THRESHOLD_MS = 20;
private final long detectionInterval;
private volatile boolean shouldRun;
private OutlierDetector slowDiskDetector;
@ -61,11 +61,27 @@ public class DataNodeDiskMetrics {
// code, status should not be overridden by daemon thread.
private boolean overrideStatus = true;
public DataNodeDiskMetrics(DataNode dn, long diskOutlierDetectionIntervalMs) {
/**
* Minimum number of disks to run outlier detection.
*/
private final long minOutlierDetectionDisks;
/**
* Threshold in milliseconds below which a disk is definitely not slow.
*/
private final long lowThresholdMs;
public DataNodeDiskMetrics(DataNode dn, long diskOutlierDetectionIntervalMs,
Configuration conf) {
this.dn = dn;
this.detectionInterval = diskOutlierDetectionIntervalMs;
slowDiskDetector = new OutlierDetector(MIN_OUTLIER_DETECTION_DISKS,
SLOW_DISK_LOW_THRESHOLD_MS);
minOutlierDetectionDisks =
conf.getLong(DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY,
DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_DEFAULT);
lowThresholdMs =
conf.getLong(DFSConfigKeys.DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY,
DFSConfigKeys.DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_DEFAULT);
slowDiskDetector =
new OutlierDetector(minOutlierDetectionDisks, lowThresholdMs);
shouldRun = true;
startDiskOutlierDetectionThread();
}

16
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
View File

@ -2374,6 +2374,22 @@
</description>
</property>
<property>
<name>dfs.datanode.min.outlier.detection.disks</name>
<value>5</value>
<description>
Minimum number of disks to run outlier detection.
</description>
</property>
<property>
<name>dfs.datanode.slowdisk.low.threshold.ms</name>
<value>20</value>
<description>
Threshold in milliseconds below which a disk is definitely not slow.
</description>
</property>
<property>
<name>hadoop.user.group.metrics.percentiles.intervals</name>
<value></value>