big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-5341. Reduce fsdataset lock duration during directory scanning. Contributed by Qus-Jiawei.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1535188 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Kihwal Lee 2013-10-23 21:29:51 +00:00
parent 1387fb1394
commit bf3271bd2b
2 changed files with 14 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -342,6 +342,9 @@ Release 2.3.0 - UNRELEASED
HDFS-5239. Allow FSNamesystem lock fairness to be configurable (daryn)
HDFS-5341. Reduce fsdataset lock duration during directory scanning.
(Qus-Jiawei via kihwal)
BUG FIXES
HDFS-5034. Remove debug prints from GetFileLinkInfo (Andrew Wang via Colin
Patrick McCabe)

12
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
View File

@ -191,6 +191,11 @@ static class ScanInfo implements Comparable<ScanInfo> {
private final FsVolumeSpi volume;
/**
* Get the file's length in async block scan
*/
private final long blockFileLength;
private final static Pattern CONDENSED_PATH_REGEX =
Pattern.compile("(?<!^)(\\\\|/){2,}");
@ -235,6 +240,7 @@ private static String getSuffix(File f, String prefix) {
getCondensedPath(vol.getBasePath());
this.blockSuffix = blockFile == null ? null :
getSuffix(blockFile, condensedVolPath);
this.blockFileLength = (blockFile != null) ? blockFile.length() : 0;
if (metaFile == null) {
this.metaSuffix = null;
} else if (blockFile == null) {
@ -251,6 +257,10 @@ File getBlockFile() {
new File(volume.getBasePath(), blockSuffix);
}
long getBlockFileLength() {
return blockFileLength;
}
File getMetaFile() {
if (metaSuffix == null) {
return null;
@ -458,7 +468,7 @@ void scan() {
// Block metadata file exits and block file is missing
addDifference(diffRecord, statsRecord, info);
} else if (info.getGenStamp() != memBlock.getGenerationStamp()
|| info.getBlockFile().length() != memBlock.getNumBytes()) {
|| info.getBlockFileLength() != memBlock.getNumBytes()) {
// Block metadata file is missing or has wrong generation stamp,
// or block file length is different than expected
statsRecord.mismatchBlocks++;