big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-5790. LeaseManager.findPath is very slow when many leases need recovery. Contributed by Todd Lipcon.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1562970 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Todd Lipcon 2014-01-30 21:21:15 +00:00
parent 6ada3c7d42
commit fd2ba4911b
3 changed files with 7 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -299,6 +299,9 @@ Release 2.4.0 - UNRELEASED
OPTIMIZATIONS OPTIMIZATIONS
HDFS-5790. LeaseManager.findPath is very slow when many leases need recovery
(todd)
BUG FIXES BUG FIXES
HDFS-5492. Port HDFS-2069 (Incorrect default trash interval in the HDFS-5492. Port HDFS-2069 (Incorrect default trash interval in the

8
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
View File

@ -3800,7 +3800,7 @@ private void commitOrCompleteLastBlock(final INodeFile fileINode,
final long diff = fileINode.getPreferredBlockSize() - commitBlock.getNumBytes(); final long diff = fileINode.getPreferredBlockSize() - commitBlock.getNumBytes();
if (diff > 0) { if (diff > 0) {
try { try {
String path = leaseManager.findPath(fileINode); String path = fileINode.getFullPathName();
dir.updateSpaceConsumed(path, 0, -diff*fileINode.getFileReplication()); dir.updateSpaceConsumed(path, 0, -diff*fileINode.getFileReplication());
} catch (IOException e) { } catch (IOException e) {
LOG.warn("Unexpected exception while updating disk space.", e); LOG.warn("Unexpected exception while updating disk space.", e);
@ -4002,7 +4002,7 @@ void commitBlockSynchronization(ExtendedBlock lastblock,
@VisibleForTesting @VisibleForTesting
String closeFileCommitBlocks(INodeFile pendingFile, BlockInfo storedBlock) String closeFileCommitBlocks(INodeFile pendingFile, BlockInfo storedBlock)
throws IOException { throws IOException {
String src = leaseManager.findPath(pendingFile); String src = pendingFile.getFullPathName();
// commit the last block and complete it if it has minimum replicas // commit the last block and complete it if it has minimum replicas
commitOrCompleteLastBlock(pendingFile, storedBlock); commitOrCompleteLastBlock(pendingFile, storedBlock);
@ -4024,7 +4024,7 @@ String closeFileCommitBlocks(INodeFile pendingFile, BlockInfo storedBlock)
@VisibleForTesting @VisibleForTesting
String persistBlocks(INodeFile pendingFile, boolean logRetryCache) String persistBlocks(INodeFile pendingFile, boolean logRetryCache)
throws IOException { throws IOException {
String src = leaseManager.findPath(pendingFile); String src = pendingFile.getFullPathName();
dir.persistBlocks(src, pendingFile, logRetryCache); dir.persistBlocks(src, pendingFile, logRetryCache);
return src; return src;
} }
@ -5952,7 +5952,7 @@ private void updatePipelineInternal(String clientName, ExtendedBlock oldBlock,
.getDatanodeStorageInfos(newNodes, newStorageIDs); .getDatanodeStorageInfos(newNodes, newStorageIDs);
blockinfo.setExpectedLocations(storages); blockinfo.setExpectedLocations(storages);
String src = leaseManager.findPath(pendingFile); String src = pendingFile.getFullPathName();
dir.persistBlocks(src, pendingFile, logRetryCache); dir.persistBlocks(src, pendingFile, logRetryCache);
} }

36
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
View File

@ -179,24 +179,6 @@ synchronized Lease reassignLease(Lease lease, String src, String newHolder) {
return addLease(newHolder, src); return addLease(newHolder, src);
} }
/**
* Finds the pathname for the specified pendingFile
*/
public synchronized String findPath(INodeFile pendingFile)
throws IOException {
FileUnderConstructionFeature uc = pendingFile.getFileUnderConstructionFeature();
Preconditions.checkArgument(uc != null);
Lease lease = getLease(uc.getClientName());
if (lease != null) {
String src = lease.findPath(pendingFile);
if (src != null) {
return src;
}
}
throw new IOException("pendingFile (=" + pendingFile + ") not found."
+ "(lease=" + lease + ")");
}
/** /**
* Renew the lease(s) held by the given client * Renew the lease(s) held by the given client
*/ */
@ -252,24 +234,6 @@ public boolean expiredSoftLimit() {
return now() - lastUpdate > softLimit; return now() - lastUpdate > softLimit;
} }
/**
* @return the path associated with the pendingFile and null if not found.
*/
private String findPath(INodeFile pendingFile) {
try {
for (String src : paths) {
INode node = fsnamesystem.dir.getINode(src);
if (node == pendingFile
|| (node.isFile() && node.asFile() == pendingFile)) {
return src;
}
}
} catch (UnresolvedLinkException e) {
throw new AssertionError("Lease files should reside on this FS");
}
return null;
}
/** Does this lease contain any path? */ /** Does this lease contain any path? */
boolean hasPath() {return !paths.isEmpty();} boolean hasPath() {return !paths.isEmpty();}