big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-8273. FSNamesystem#Delete() should not call logSync() when holding the lock. Contributed by Haohui Mai.

Browse Source
This commit is contained in:
Haohui Mai 2015-04-28 18:05:46 -07:00
parent 519092322d
commit c79e7f7d99
3 changed files with 16 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -627,6 +627,9 @@ Release 2.7.1 - UNRELEASED
HDFS-8070. Pre-HDFS-7915 DFSClient cannot use short circuit on HDFS-8070. Pre-HDFS-7915 DFSClient cannot use short circuit on
post-HDFS-7915 DataNode (cmccabe) post-HDFS-7915 DataNode (cmccabe)
HDFS-8273. FSNamesystem#Delete() should not call logSync() when holding the
lock. (wheat9)
Release 2.7.0 - 2015-04-20 Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

1
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java
View File

@ -175,7 +175,6 @@ static BlocksMapUpdateInfo deleteInternal(
incrDeletedFileCount(filesRemoved); incrDeletedFileCount(filesRemoved);
fsn.removeLeasesAndINodes(src, removedINodes, true); fsn.removeLeasesAndINodes(src, removedINodes, true);
fsd.getEditLog().logSync();
if (NameNode.stateChangeLog.isDebugEnabled()) { if (NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("DIR* Namesystem.delete: " NameNode.stateChangeLog.debug("DIR* Namesystem.delete: "

16
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
View File

@ -3690,6 +3690,7 @@ boolean delete(String src, boolean recursive, boolean logRetryCache)
} finally { } finally {
writeUnlock(); writeUnlock();
} }
getEditLog().logSync();
if (toRemovedBlocks != null) { if (toRemovedBlocks != null) {
removeBlocks(toRemovedBlocks); // Incremental deletion of blocks removeBlocks(toRemovedBlocks); // Incremental deletion of blocks
} }
@ -4695,22 +4696,21 @@ public LazyPersistFileScrubber(final int scrubIntervalSec) {
* blocks and unlink them from the namespace. * blocks and unlink them from the namespace.
*/ */
private void clearCorruptLazyPersistFiles() private void clearCorruptLazyPersistFiles()
throws SafeModeException, AccessControlException, throws IOException {
UnresolvedLinkException, IOException {
BlockStoragePolicy lpPolicy = blockManager.getStoragePolicy("LAZY_PERSIST"); BlockStoragePolicy lpPolicy = blockManager.getStoragePolicy("LAZY_PERSIST");
List<BlockCollection> filesToDelete = new ArrayList<BlockCollection>(); List<BlockCollection> filesToDelete = new ArrayList<>();
boolean changed = false;
writeLock(); writeLock();
try { try {
final Iterator<Block> it = blockManager.getCorruptReplicaBlockIterator(); final Iterator<Block> it = blockManager.getCorruptReplicaBlockIterator();
while (it.hasNext()) { while (it.hasNext()) {
Block b = it.next(); Block b = it.next();
BlockInfoContiguous blockInfo = blockManager.getStoredBlock(b); BlockInfoContiguous blockInfo = blockManager.getStoredBlock(b);
if (blockInfo.getBlockCollection().getStoragePolicyID() == lpPolicy.getId()) { if (blockInfo.getBlockCollection().getStoragePolicyID()
== lpPolicy.getId()) {
filesToDelete.add(blockInfo.getBlockCollection()); filesToDelete.add(blockInfo.getBlockCollection());
} }
} }
@ -4721,6 +4721,7 @@ private void clearCorruptLazyPersistFiles()
FSDirDeleteOp.deleteInternal( FSDirDeleteOp.deleteInternal(
FSNamesystem.this, bc.getName(), FSNamesystem.this, bc.getName(),
INodesInPath.fromINode((INodeFile) bc), false); INodesInPath.fromINode((INodeFile) bc), false);
changed |= toRemoveBlocks != null;
if (toRemoveBlocks != null) { if (toRemoveBlocks != null) {
removeBlocks(toRemoveBlocks); // Incremental deletion of blocks removeBlocks(toRemoveBlocks); // Incremental deletion of blocks
} }
@ -4728,6 +4729,9 @@ private void clearCorruptLazyPersistFiles()
} finally { } finally {
writeUnlock(); writeUnlock();
} }
if (changed) {
getEditLog().logSync();
}
} }
@Override @Override