big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-7819. Log WARN message for the blocks which are not in Block ID based layout (Rakesh R via Colin P. McCabe)

Browse Source
This commit is contained in:
Colin Patrick Mccabe 2015-02-26 11:58:29 -08:00
parent dce8b9c4d0
commit f0c980abed
2 changed files with 25 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -679,6 +679,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7832. Show 'Last Modified' in Namenode's 'Browse Filesystem' HDFS-7832. Show 'Last Modified' in Namenode's 'Browse Filesystem'
(vinayakumarb) (vinayakumarb)
HDFS-7819. Log WARN message for the blocks which are not in Block ID based
layout (Rakesh R via Colin P. McCabe)
OPTIMIZATIONS OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode. HDFS-7454. Reduce memory footprint for AclEntries in NameNode.

26
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
View File

@ -597,14 +597,15 @@ public ScanInfoPerBlockPool call() throws Exception {
for (String bpid : bpList) { for (String bpid : bpList) {
LinkedList<ScanInfo> report = new LinkedList<ScanInfo>(); LinkedList<ScanInfo> report = new LinkedList<ScanInfo>();
File bpFinalizedDir = volume.getFinalizedDir(bpid); File bpFinalizedDir = volume.getFinalizedDir(bpid);
result.put(bpid, compileReport(volume, bpFinalizedDir, report)); result.put(bpid,
compileReport(volume, bpFinalizedDir, bpFinalizedDir, report));
} }
return result; return result;
} }
/** Compile list {@link ScanInfo} for the blocks in the directory <dir> */ /** Compile list {@link ScanInfo} for the blocks in the directory <dir> */
private LinkedList<ScanInfo> compileReport(FsVolumeSpi vol, File dir, private LinkedList<ScanInfo> compileReport(FsVolumeSpi vol,
LinkedList<ScanInfo> report) { File bpFinalizedDir, File dir, LinkedList<ScanInfo> report) {
File[] files; File[] files;
try { try {
files = FileUtil.listFiles(dir); files = FileUtil.listFiles(dir);
@ -622,12 +623,14 @@ private LinkedList<ScanInfo> compileReport(FsVolumeSpi vol, File dir,
*/ */
for (int i = 0; i < files.length; i++) { for (int i = 0; i < files.length; i++) {
if (files[i].isDirectory()) { if (files[i].isDirectory()) {
compileReport(vol, files[i], report); compileReport(vol, bpFinalizedDir, files[i], report);
continue; continue;
} }
if (!Block.isBlockFilename(files[i])) { if (!Block.isBlockFilename(files[i])) {
if (isBlockMetaFile("blk_", files[i].getName())) { if (isBlockMetaFile("blk_", files[i].getName())) {
long blockId = Block.getBlockId(files[i].getName()); long blockId = Block.getBlockId(files[i].getName());
verifyFileLocation(files[i].getParentFile(), bpFinalizedDir,
blockId);
report.add(new ScanInfo(blockId, null, files[i], vol)); report.add(new ScanInfo(blockId, null, files[i], vol));
} }
continue; continue;
@ -646,9 +649,24 @@ private LinkedList<ScanInfo> compileReport(FsVolumeSpi vol, File dir,
break; break;
} }
} }
verifyFileLocation(blockFile.getParentFile(), bpFinalizedDir,
blockId);
report.add(new ScanInfo(blockId, blockFile, metaFile, vol)); report.add(new ScanInfo(blockId, blockFile, metaFile, vol));
} }
return report; return report;
} }
/**
* Verify whether the actual directory location of block file has the
* expected directory path computed using its block ID.
*/
private void verifyFileLocation(File actualBlockDir,
File bpFinalizedDir, long blockId) {
File blockDir = DatanodeUtil.idToBlockDir(bpFinalizedDir, blockId);
if (actualBlockDir.compareTo(blockDir) != 0) {
LOG.warn("Block: " + blockId
+ " has to be upgraded to block ID-based layout");
}
}
} }
} }