big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-10975. fsck -list-corruptfileblocks does not report corrupt EC files. Contributed by Takanobu Asanuma.

Browse Source
This commit is contained in:
Wei-Chiu Chuang 2016-10-21 15:43:12 -07:00
parent be72372248
commit df857f0d10
2 changed files with 14 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
View File

@ -660,6 +660,13 @@ private void collectBlocksSummary(String parent, HdfsFileStatus file,
decommissioningReplicas;
res.totalReplicas += totalReplicasPerBlock;
boolean isMissing;
if (storedBlock.isStriped()) {
isMissing = totalReplicasPerBlock < minReplication;
} else {
isMissing = totalReplicasPerBlock == 0;
}
// count expected replicas
short targetFileReplication;
if (file.getErasureCodingPolicy() != null) {
@ -697,7 +704,7 @@ private void collectBlocksSummary(String parent, HdfsFileStatus file,
res.numMinReplicatedBlocks++;
// count missing replicas / under replicated blocks
if (totalReplicasPerBlock < targetFileReplication && totalReplicasPerBlock > 0) {
if (totalReplicasPerBlock < targetFileReplication && !isMissing) {
res.missingReplicas += (targetFileReplication - totalReplicasPerBlock);
res.numUnderReplicatedBlocks += 1;
underReplicatedPerFile++;
@ -737,12 +744,6 @@ private void collectBlocksSummary(String parent, HdfsFileStatus file,
String blkName = block.toString();
report.append(blockNumber + ". " + blkName + " len=" +
block.getNumBytes());
boolean isMissing;
if (storedBlock.isStriped()) {
isMissing = totalReplicasPerBlock < minReplication;
} else {
isMissing = totalReplicasPerBlock == 0;
}
if (isMissing && !isCorrupt) {
// If the block is corrupted, it means all its available replicas are
// corrupted in the case of replication, and it means the state of the

6
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
View File

@ -2013,6 +2013,9 @@ public void testFsckCorruptECFile() throws Exception {
String outStr = runFsck(conf, 1, true, "/");
assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS));
assertTrue(outStr.contains("Under-erasure-coded block groups:\t0"));
outStr = runFsck(conf, -1, true, "/", "-list-corruptfileblocks");
assertTrue(outStr.contains("has 1 CORRUPT files"));
}
@Test (timeout = 300000)
@ -2053,6 +2056,9 @@ public void testFsckMissingECFile() throws Exception {
"-locations");
assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS));
assertTrue(outStr.contains("Live_repl=" + (dataBlocks - 1)));
assertTrue(outStr.contains("Under-erasure-coded block groups:\t0"));
outStr = runFsck(conf, -1, true, "/", "-list-corruptfileblocks");
assertTrue(outStr.contains("has 1 CORRUPT files"));
}
private void waitForUnrecoverableBlockGroup(Configuration configuration)