big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-13831. Make block increment deletion number configurable. Contributed by Ryan Wu.

Browse Source
This commit is contained in:
Yiqun Lin 2018-08-27 14:55:46 +08:00
parent a4121c71c2
commit b9b964d253
4 changed files with 25 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
View File

@ -395,6 +395,11 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY = "dfs.namenode.startup.delay.block.deletion.sec";
public static final long DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT = 0L;
/** Block deletion increment. */
public static final String DFS_NAMENODE_BLOCK_DELETION_INCREMENT_KEY =
"dfs.namenode.block.deletion.increment";
public static final int DFS_NAMENODE_BLOCK_DELETION_INCREMENT_DEFAULT = 1000;
public static final String DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES =
HdfsClientConfigKeys.DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES;
public static final boolean DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES_DEFAULT =

11
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
View File

@ -428,12 +428,12 @@ private void logAuditEvent(boolean succeeded,
FSNamesystem.class.getName() + ".audit");
private final int maxCorruptFileBlocksReturn;
static int BLOCK_DELETION_INCREMENT = 1000;
private final boolean isPermissionEnabled;
private final UserGroupInformation fsOwner;
private final String supergroup;
private final boolean standbyShouldCheckpoint;
private final int snapshotDiffReportLimit;
private final int blockDeletionIncrement;
/** Interval between each check of lease to release. */
private final long leaseRecheckIntervalMs;
@ -909,6 +909,13 @@ static FSNamesystem loadFromDisk(Configuration conf) throws IOException {
DFSConfigKeys.DFS_NAMENODE_LIST_OPENFILES_NUM_RESPONSES +
" must be a positive integer."
);
this.blockDeletionIncrement = conf.getInt(
DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_INCREMENT_KEY,
DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_INCREMENT_DEFAULT);
Preconditions.checkArgument(blockDeletionIncrement > 0,
DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_INCREMENT_KEY +
" must be a positive integer.");
} catch(IOException e) {
LOG.error(getClass().getSimpleName() + " initialization failed.", e);
close();
@ -3094,7 +3101,7 @@ void removeBlocks(BlocksMapUpdateInfo blocks) {
while (iter.hasNext()) {
writeLock();
try {
for (int i = 0; i < BLOCK_DELETION_INCREMENT && iter.hasNext(); i++) {
for (int i = 0; i < blockDeletionIncrement && iter.hasNext(); i++) {
blockManager.removeBlock(iter.next());
}
} finally {

10
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
View File

@ -5160,4 +5160,14 @@
will throw NameNodeFormatException.
</description>
</property>
<property>
<name>dfs.namenode.block.deletion.increment</name>
<value>1000</value>
<description>
The number of block deletion increment.
This setting will control the block increment deletion rate to
ensure that other waiters on the lock can get in.
</description>
</property>
</configuration>

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLargeDirectoryDelete.java
View File

@ -49,6 +49,7 @@ public class TestLargeDirectoryDelete {
static {
CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1);
CONF.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 1);
CONF.setInt(DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_INCREMENT_KEY, 1);
}
/** create a file with a length of <code>filelen</code> */
@ -137,7 +138,6 @@ protected void execute() throws Throwable {
threads[1].start();
final long start = Time.now();
FSNamesystem.BLOCK_DELETION_INCREMENT = 1;
mc.getFileSystem().delete(new Path("/root"), true); // recursive delete
final long end = Time.now();
threads[0].endThread();