big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-17063. Support to configure different capacity reserved for each disk of DataNode. (#5793). Contributed by QI Jiale.

Browse Source 
Reviewed-by : Tao Li <tomscut@apache.org>
Signed-off-by: He Xiaoqiao <hexiaoqiao@apache.org>
This commit is contained in:
QI Jiale 2023-11-17 11:59:04 +08:00 committed by GitHub
parent 469f20a52b
commit 28068aa320
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 104 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
View File

@ -171,7 +171,8 @@ public class FsVolumeImpl implements FsVolumeSpi {
this.usage = usage;
if (this.usage != null) {
reserved = new ReservedSpaceCalculator.Builder(conf)
.setUsage(this.usage).setStorageType(storageType).build();
.setUsage(this.usage).setStorageType(storageType)
.setDir(currentDir != null ? currentDir.getParent() : "NULL").build();
boolean fixedSizeVolume = conf.getBoolean(
DFSConfigKeys.DFS_DATANODE_FIXED_VOLUME_SIZE_KEY,
DFSConfigKeys.DFS_DATANODE_FIXED_VOLUME_SIZE_DEFAULT);

43
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/ReservedSpaceCalculator.java
View File

@ -46,6 +46,8 @@ public static class Builder {
private DF usage;
private StorageType storageType;
private String dir;
public Builder(Configuration conf) {
this.conf = conf;
}
@ -61,6 +63,11 @@ public Builder setStorageType(
return this;
}
public Builder setDir(String newDir) {
this.dir = newDir;
return this;
}
ReservedSpaceCalculator build() {
try {
Class<? extends ReservedSpaceCalculator> clazz = conf.getClass(
@ -69,10 +76,10 @@ ReservedSpaceCalculator build() {
ReservedSpaceCalculator.class);
Constructor constructor = clazz.getConstructor(
Configuration.class, DF.class, StorageType.class);
Configuration.class, DF.class, StorageType.class, String.class);
return (ReservedSpaceCalculator) constructor.newInstance(
conf, usage, storageType);
conf, usage, storageType, dir);
} catch (Exception e) {
throw new IllegalStateException(
"Error instantiating ReservedSpaceCalculator", e);
@ -84,20 +91,30 @@ ReservedSpaceCalculator build() {
private final Configuration conf;
private final StorageType storageType;
private final String dir;
ReservedSpaceCalculator(Configuration conf, DF usage,
StorageType storageType) {
StorageType storageType, String dir) {
this.usage = usage;
this.conf = conf;
this.storageType = storageType;
this.dir = dir;
}
DF getUsage() {
return usage;
}
String getDir() {
return dir;
}
long getReservedFromConf(String key, long defaultValue) {
return conf.getLong(key + "." + StringUtils.toLowerCase(
storageType.toString()), conf.getLongBytes(key, defaultValue));
return conf.getLong(
key + "." + getDir() + "." + StringUtils.toLowerCase(storageType.toString()),
conf.getLong(key + "." + getDir(),
conf.getLong(key + "." + StringUtils.toLowerCase(storageType.toString()),
conf.getLongBytes(key, defaultValue))));
}
/**
@ -117,8 +134,8 @@ public static class ReservedSpaceCalculatorAbsolute extends
private final long reservedBytes;
public ReservedSpaceCalculatorAbsolute(Configuration conf, DF usage,
StorageType storageType) {
super(conf, usage, storageType);
StorageType storageType, String dir) {
super(conf, usage, storageType, dir);
this.reservedBytes = getReservedFromConf(DFS_DATANODE_DU_RESERVED_KEY,
DFS_DATANODE_DU_RESERVED_DEFAULT);
}
@ -138,8 +155,8 @@ public static class ReservedSpaceCalculatorPercentage extends
private final long reservedPct;
public ReservedSpaceCalculatorPercentage(Configuration conf, DF usage,
StorageType storageType) {
super(conf, usage, storageType);
StorageType storageType, String dir) {
super(conf, usage, storageType, dir);
this.reservedPct = getReservedFromConf(
DFS_DATANODE_DU_RESERVED_PERCENTAGE_KEY,
DFS_DATANODE_DU_RESERVED_PERCENTAGE_DEFAULT);
@ -162,8 +179,8 @@ public static class ReservedSpaceCalculatorConservative extends
private final long reservedPct;
public ReservedSpaceCalculatorConservative(Configuration conf, DF usage,
StorageType storageType) {
super(conf, usage, storageType);
StorageType storageType, String dir) {
super(conf, usage, storageType, dir);
this.reservedBytes = getReservedFromConf(DFS_DATANODE_DU_RESERVED_KEY,
DFS_DATANODE_DU_RESERVED_DEFAULT);
this.reservedPct = getReservedFromConf(
@ -197,8 +214,8 @@ public static class ReservedSpaceCalculatorAggressive extends
private final long reservedPct;
public ReservedSpaceCalculatorAggressive(Configuration conf, DF usage,
StorageType storageType) {
super(conf, usage, storageType);
StorageType storageType, String dir) {
super(conf, usage, storageType, dir);
this.reservedBytes = getReservedFromConf(DFS_DATANODE_DU_RESERVED_KEY,
DFS_DATANODE_DU_RESERVED_DEFAULT);
this.reservedPct = getReservedFromConf(

20
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
View File

@ -397,12 +397,19 @@
<name>dfs.datanode.du.reserved</name>
<value>0</value>
<description>Reserved space in bytes per volume. Always leave this much space free for non dfs use.
Specific directory based reservation is supported. The property can be followed with directory
name which is set at 'dfs.datanode.data.dir'. For example, reserved space for /data/hdfs1/data
can be configured using property 'dfs.datanode.du.reserved./data/hdfs1/data'. If specific directory
reservation is not configured then dfs.datanode.du.reserved will be used.
Specific storage type based reservation is also supported. The property can be followed with
corresponding storage types ([ssd]/[disk]/[archive]/[ram_disk]/[nvdimm]) for cluster with heterogeneous storage.
For example, reserved space for RAM_DISK storage can be configured using property
'dfs.datanode.du.reserved.ram_disk'. If specific storage type reservation is not configured
then dfs.datanode.du.reserved will be used. Support multiple size unit suffix(case insensitive),
as described in dfs.blocksize.
as described in dfs.blocksize. Use directory name and storage type based reservation at the
same time is also allowed if both are configured.
Property priority example: dfs.datanode.du.reserved./data/hdfs1/data.ram_disk >
dfs.datanode.du.reserved./data/hdfs1/data > dfs.datanode.du.reserved.ram_disk > dfs.datanode.du.reserved
Note: In case of using tune2fs to set reserved-blocks-percentage, or other filesystem tools,
then you can possibly run into out of disk errors because hadoop will not check those
external tool configurations.
@ -414,12 +421,19 @@
<value>0</value>
<description>Reserved space in percentage. Read dfs.datanode.du.reserved.calculator to see
when this takes effect. The actual number of bytes reserved will be calculated by using the
total capacity of the data directory in question. Specific storage type based reservation
total capacity of the data directory in question. Specific directory based reservation is
supported. The property can be followed with directory name which is set at 'dfs.datanode.data.dir'.
For example, reserved percentage space for /data/hdfs1/data can be configured using property
'dfs.datanode.du.reserved.pct./data/hdfs1/data'. If specific directory reservation is not
configured then dfs.datanode.du.reserved.pct will be used. Specific storage type based reservation
is also supported. The property can be followed with corresponding storage types
([ssd]/[disk]/[archive]/[ram_disk]/[nvdimm]) for cluster with heterogeneous storage.
For example, reserved percentage space for RAM_DISK storage can be configured using property
'dfs.datanode.du.reserved.pct.ram_disk'. If specific storage type reservation is not configured
then dfs.datanode.du.reserved.pct will be used.
then dfs.datanode.du.reserved.pct will be used. Use directory and storage type based reservation
is also allowed if both are configured.
Priority example: dfs.datanode.du.reserved.pct./data/hdfs1/data.ram_disk > dfs.datanode.du.reserved.pct./data/hdfs1/data
> dfs.datanode.du.reserved.pct.ram_disk > dfs.datanode.du.reserved.pct
</description>
</property>

56
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestReservedSpaceCalculator.java
View File

@ -168,6 +168,55 @@ public void testReservedSpaceAggresivePerStorageType() {
checkReserved(StorageType.ARCHIVE, 100000, 5000);
}
@Test
public void testReservedSpaceAbsolutePerDir() {
conf.setClass(DFS_DATANODE_DU_RESERVED_CALCULATOR_KEY, ReservedSpaceCalculatorAbsolute.class,
ReservedSpaceCalculator.class);
String dir1 = "/data/hdfs1/data";
String dir2 = "/data/hdfs2/data";
String dir3 = "/data/hdfs3/data";
conf.setLong(DFS_DATANODE_DU_RESERVED_KEY + "." + dir1 + ".ssd", 900);
conf.setLong(DFS_DATANODE_DU_RESERVED_KEY + "." + dir1, 1800);
conf.setLong(DFS_DATANODE_DU_RESERVED_KEY + "." + dir2, 2700);
conf.setLong(DFS_DATANODE_DU_RESERVED_KEY + ".ssd", 3600);
conf.setLong(DFS_DATANODE_DU_RESERVED_KEY, 4500);
checkReserved(StorageType.SSD, 10000, 900, dir1);
checkReserved(StorageType.DISK, 10000, 1800, dir1);
checkReserved(StorageType.SSD, 10000, 2700, dir2);
checkReserved(StorageType.DISK, 10000, 2700, dir2);
checkReserved(StorageType.SSD, 10000, 3600, dir3);
checkReserved(StorageType.DISK, 10000, 4500, dir3);
}
@Test
public void testReservedSpacePercentagePerDir() {
conf.setClass(DFS_DATANODE_DU_RESERVED_CALCULATOR_KEY,
ReservedSpaceCalculatorPercentage.class,
ReservedSpaceCalculator.class);
String dir1 = "/data/hdfs1/data";
String dir2 = "/data/hdfs2/data";
String dir3 = "/data/hdfs3/data";
// Set percentage reserved values for different directories
conf.setLong(DFS_DATANODE_DU_RESERVED_PERCENTAGE_KEY + "." + dir1 + ".ssd", 20);
conf.setLong(DFS_DATANODE_DU_RESERVED_PERCENTAGE_KEY + "." + dir1, 10);
conf.setLong(DFS_DATANODE_DU_RESERVED_PERCENTAGE_KEY + "." + dir2, 25);
conf.setLong(DFS_DATANODE_DU_RESERVED_PERCENTAGE_KEY + ".ssd", 30);
conf.setLong(DFS_DATANODE_DU_RESERVED_PERCENTAGE_KEY, 40);
// Verify reserved space calculations for different directories and storage types
checkReserved(StorageType.SSD, 10000, 2000, dir1);
checkReserved(StorageType.DISK, 10000, 1000, dir1);
checkReserved(StorageType.SSD, 10000, 2500, dir2);
checkReserved(StorageType.DISK, 10000, 2500, dir2);
checkReserved(StorageType.SSD, 10000, 3000, dir3);
checkReserved(StorageType.DISK, 10000, 4000, dir3);
}
@Test(expected = IllegalStateException.class)
public void testInvalidCalculator() {
conf.set(DFS_DATANODE_DU_RESERVED_CALCULATOR_KEY, "INVALIDTYPE");
@ -179,10 +228,15 @@ public void testInvalidCalculator() {
private void checkReserved(StorageType storageType,
long totalCapacity, long reservedExpected) {
checkReserved(storageType, totalCapacity, reservedExpected, "NULL");
}
private void checkReserved(StorageType storageType,
long totalCapacity, long reservedExpected, String dir) {
when(usage.getCapacity()).thenReturn(totalCapacity);
reserved = new ReservedSpaceCalculator.Builder(conf).setUsage(usage)
.setStorageType(storageType).build();
.setStorageType(storageType).setDir(dir).build();
assertEquals(reservedExpected, reserved.getReserved());
}
}