diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index c377e77106..5f8a26daf4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -843,6 +843,9 @@ Release 0.23.0 - Unreleased HDFS-2118. Couple dfs data dir improvements. (eli) + HDFS-2500. Avoid file system operations in BPOfferService thread while + processing deletes. (todd) + BUG FIXES HDFS-2347. Fix checkpointTxnCount's comment about editlog size. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 7310d1a20c..0ec544d9a8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -1151,8 +1151,15 @@ private void offerService() throws Exception { if (!heartbeatsDisabledForTests) { DatanodeCommand[] cmds = sendHeartBeat(); metrics.addHeartbeat(now() - startTime); + + long startProcessCommands = now(); if (!processCommand(cmds)) continue; + long endProcessCommands = now(); + if (endProcessCommands - startProcessCommands > 2000) { + LOG.info("Took " + (endProcessCommands - startProcessCommands) + + "ms to process " + cmds.length + " commands from NN"); + } } } if (pendingReceivedRequests > 0 diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java index 117663cd86..512d0b64bb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FSDataset.java @@ -2087,10 +2087,9 @@ public void invalidate(String bpid, Block invalidBlks[]) throws IOException { volumeMap.remove(bpid, invalidBlks[i]); } File metaFile = getMetaFile(f, invalidBlks[i].getGenerationStamp()); - long dfsBytes = f.length() + metaFile.length(); // Delete the block asynchronously to make sure we can do it fast enough - asyncDiskService.deleteAsync(v, f, metaFile, dfsBytes, + asyncDiskService.deleteAsync(v, f, metaFile, new ExtendedBlock(bpid, invalidBlks[i])); } if (error) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FSDatasetAsyncDiskService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FSDatasetAsyncDiskService.java index 4b49b05eea..408a6afc47 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FSDatasetAsyncDiskService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FSDatasetAsyncDiskService.java @@ -152,11 +152,11 @@ synchronized void shutdown() { * dfsUsed statistics accordingly. */ void deleteAsync(FSDataset.FSVolume volume, File blockFile, File metaFile, - long dfsBytes, ExtendedBlock block) { + ExtendedBlock block) { DataNode.LOG.info("Scheduling block " + block.getLocalBlock().toString() + " file " + blockFile + " for deletion"); ReplicaFileDeleteTask deletionTask = new ReplicaFileDeleteTask(dataset, - volume, blockFile, metaFile, dfsBytes, block); + volume, blockFile, metaFile, block); execute(volume.getCurrentDir(), deletionTask); } @@ -168,16 +168,14 @@ static class ReplicaFileDeleteTask implements Runnable { final FSDataset.FSVolume volume; final File blockFile; final File metaFile; - final long dfsBytes; final ExtendedBlock block; ReplicaFileDeleteTask(FSDataset dataset, FSDataset.FSVolume volume, File blockFile, - File metaFile, long dfsBytes, ExtendedBlock block) { + File metaFile, ExtendedBlock block) { this.dataset = dataset; this.volume = volume; this.blockFile = blockFile; this.metaFile = metaFile; - this.dfsBytes = dfsBytes; this.block = block; } @@ -195,6 +193,7 @@ public String toString() { @Override public void run() { + long dfsBytes = blockFile.length() + metaFile.length(); if ( !blockFile.delete() || ( !metaFile.delete() && metaFile.exists() ) ) { DataNode.LOG.warn("Unexpected error trying to delete block " + block.getBlockPoolId() + " " + block.getLocalBlock().toString()