big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-17171. CONGESTION_RATIO should be configurable (#5996)

Browse Source 
Reviewed-by: Ayush Saxena <ayushsaxena@apache.org>
Signed-off-by: Tao Li <tomscut@apache.org>
This commit is contained in:
hfutatzhanghb 2023-10-08 10:36:09 +08:00 committed by GitHub
parent 42b32fbbdc
commit ea3cb12ec8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 20 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
View File

@ -1573,6 +1573,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final boolean DFS_PIPELINE_ECN_ENABLED_DEFAULT = false; public static final boolean DFS_PIPELINE_ECN_ENABLED_DEFAULT = false;
public static final String DFS_PIPELINE_SLOWNODE_ENABLED = "dfs.pipeline.slownode"; public static final String DFS_PIPELINE_SLOWNODE_ENABLED = "dfs.pipeline.slownode";
public static final boolean DFS_PIPELINE_SLOWNODE_ENABLED_DEFAULT = false; public static final boolean DFS_PIPELINE_SLOWNODE_ENABLED_DEFAULT = false;
public static final String DFS_PIPELINE_CONGESTION_RATIO = "dfs.pipeline.congestion.ratio";
public static final double DFS_PIPELINE_CONGESTION_RATIO_DEFAULT = 1.5;
// Key Provider Cache Expiry // Key Provider Cache Expiry
public static final String DFS_DATANODE_BLOCK_PINNING_ENABLED = public static final String DFS_DATANODE_BLOCK_PINNING_ENABLED =
@ -2042,5 +2044,4 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final long DFS_LEASE_HARDLIMIT_DEFAULT = public static final long DFS_LEASE_HARDLIMIT_DEFAULT =
HdfsClientConfigKeys.DFS_LEASE_HARDLIMIT_DEFAULT; HdfsClientConfigKeys.DFS_LEASE_HARDLIMIT_DEFAULT;
} }

12
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
View File

@ -462,7 +462,7 @@ public static InetSocketAddress createSocketAddr(String target) {
private final Tracer tracer; private final Tracer tracer;
private static final int NUM_CORES = Runtime.getRuntime() private static final int NUM_CORES = Runtime.getRuntime()
.availableProcessors(); .availableProcessors();
private static final double CONGESTION_RATIO = 1.5; private final double congestionRatio;
private DiskBalancer diskBalancer; private DiskBalancer diskBalancer;
private DataSetLockManager dataSetLockManager; private DataSetLockManager dataSetLockManager;
@ -515,6 +515,10 @@ private static Tracer createTracer(Configuration conf) {
volumeChecker = new DatasetVolumeChecker(conf, new Timer()); volumeChecker = new DatasetVolumeChecker(conf, new Timer());
this.xferService = this.xferService =
HadoopExecutors.newCachedThreadPool(new Daemon.DaemonFactory()); HadoopExecutors.newCachedThreadPool(new Daemon.DaemonFactory());
double congestionRationTmp = conf.getDouble(DFSConfigKeys.DFS_PIPELINE_CONGESTION_RATIO,
DFSConfigKeys.DFS_PIPELINE_CONGESTION_RATIO_DEFAULT);
this.congestionRatio = congestionRationTmp > 0 ?
congestionRationTmp : DFSConfigKeys.DFS_PIPELINE_CONGESTION_RATIO_DEFAULT;
} }
/** /**
@ -614,6 +618,10 @@ public Map<String, Long> load(String key) {
new DataTransferThrottler(100, ecReconstuctReadBandwidth) : null; new DataTransferThrottler(100, ecReconstuctReadBandwidth) : null;
this.ecReconstuctWriteThrottler = ecReconstuctWriteBandwidth > 0 ? this.ecReconstuctWriteThrottler = ecReconstuctWriteBandwidth > 0 ?
new DataTransferThrottler(100, ecReconstuctWriteBandwidth) : null; new DataTransferThrottler(100, ecReconstuctWriteBandwidth) : null;
double congestionRationTmp = conf.getDouble(DFSConfigKeys.DFS_PIPELINE_CONGESTION_RATIO,
DFSConfigKeys.DFS_PIPELINE_CONGESTION_RATIO_DEFAULT);
this.congestionRatio = congestionRationTmp > 0 ?
congestionRationTmp : DFSConfigKeys.DFS_PIPELINE_CONGESTION_RATIO_DEFAULT;
} }
@Override // ReconfigurableBase @Override // ReconfigurableBase
@ -1070,7 +1078,7 @@ public PipelineAck.ECN getECN() {
} }
double load = ManagementFactory.getOperatingSystemMXBean() double load = ManagementFactory.getOperatingSystemMXBean()
.getSystemLoadAverage(); .getSystemLoadAverage();
return load > NUM_CORES * CONGESTION_RATIO ? PipelineAck.ECN.CONGESTED : return load > NUM_CORES * congestionRatio ? PipelineAck.ECN.CONGESTED :
PipelineAck.ECN.SUPPORTED; PipelineAck.ECN.SUPPORTED;
} }

8
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
View File

@ -5619,6 +5619,14 @@
</description> </description>
</property> </property>
<property>
<name>dfs.pipeline.congestion.ratio</name>
<value>1.5</value>
<description>
The ratio which is used to compute congestion load.
</description>
</property>
<property> <property>
<name>dfs.qjournal.accept-recovery.timeout.ms</name> <name>dfs.qjournal.accept-recovery.timeout.ms</name>
<value>120000</value> <value>120000</value>