big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-16500. Make asynchronous blocks deletion lock and unlock durtion threshold configurable. (#4061). Contributed by Chengwei Wang.

Browse Source
This commit is contained in:
smarthan 2022-04-21 12:32:12 +08:00 committed by GitHub
parent a4683be65e
commit bf4730b4d6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 38 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
View File

@ -481,6 +481,17 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
"dfs.namenode.block.deletion.increment"; "dfs.namenode.block.deletion.increment";
public static final int DFS_NAMENODE_BLOCK_DELETION_INCREMENT_DEFAULT = 1000; public static final int DFS_NAMENODE_BLOCK_DELETION_INCREMENT_DEFAULT = 1000;
/** The limit of single lock holding duration.*/
public static final String DFS_NAMENODE_BLOCK_DELETION_LOCK_THRESHOLD_MS =
"dfs.namenode.block.deletion.lock.threshold.ms";
public static final int DFS_NAMENODE_BLOCK_DELETION_LOCK_THRESHOLD_MS_DEFAULT =
50;
/** The sleep interval for releasing lock.*/
public static final String DFS_NAMENODE_BLOCK_DELETION_UNLOCK_INTERVAL_MS =
"dfs.namenode.block.deletion.unlock.interval.ms";
public static final int DFS_NAMENODE_BLOCK_DELETION_UNLOCK_INTERVAL_MS_DEFAULT =
10;
public static final String DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES = public static final String DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES =
HdfsClientConfigKeys.DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES; HdfsClientConfigKeys.DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES;
public static final boolean DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES_DEFAULT = public static final boolean DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES_DEFAULT =

10
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
View File

@ -192,8 +192,8 @@ public class BlockManager implements BlockStatsMXBean {
private volatile long lowRedundancyBlocksCount = 0L; private volatile long lowRedundancyBlocksCount = 0L;
private volatile long scheduledReplicationBlocksCount = 0L; private volatile long scheduledReplicationBlocksCount = 0L;
private final long deleteBlockLockTimeMs = 500; private final long deleteBlockLockTimeMs;
private final long deleteBlockUnlockIntervalTimeMs = 100; private final long deleteBlockUnlockIntervalTimeMs;
/** flag indicating whether replication queues have been initialized */ /** flag indicating whether replication queues have been initialized */
private boolean initializedReplQueues; private boolean initializedReplQueues;
@ -495,6 +495,12 @@ public BlockManager(final Namesystem namesystem, boolean haEnabled,
startupDelayBlockDeletionInMs = conf.getLong( startupDelayBlockDeletionInMs = conf.getLong(
DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY, DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY,
DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT) * 1000L; DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT) * 1000L;
deleteBlockLockTimeMs = conf.getLong(
DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_LOCK_THRESHOLD_MS,
DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_LOCK_THRESHOLD_MS_DEFAULT);
deleteBlockUnlockIntervalTimeMs = conf.getLong(
DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_UNLOCK_INTERVAL_MS,
DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_UNLOCK_INTERVAL_MS_DEFAULT);
invalidateBlocks = new InvalidateBlocks( invalidateBlocks = new InvalidateBlocks(
datanodeManager.getBlockInvalidateLimit(), datanodeManager.getBlockInvalidateLimit(),
startupDelayBlockDeletionInMs, startupDelayBlockDeletionInMs,

19
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
View File

@ -6136,6 +6136,25 @@
</description> </description>
</property> </property>
<property>
<name>dfs.namenode.block.deletion.lock.threshold.ms</name>
<value>50</value>
<description>
The limit of single time lock holding duration for the block asynchronous
deletion thread.
</description>
</property>
<property>
<name>dfs.namenode.block.deletion.unlock.interval.ms</name>
<value>10</value>
<description>
The sleep interval for yield lock.
When the single time lock holding duration of the block asynchronous deletion
thread exceeds limit, sleeping to yield lock.
</description>
</property>
<property> <property>
<name>dfs.namenode.rpc-address.auxiliary-ports</name> <name>dfs.namenode.rpc-address.auxiliary-ports</name>
<value></value> <value></value>