From 9a3c2379ef24cdca5153abf4b63fde1131ff8989 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Tue, 8 Aug 2017 23:43:24 -0700 Subject: [PATCH] HDFS-12182. BlockManager.metaSave does not distinguish between "under replicated" and "missing" blocks. Contributed by Wellington Chevreuil. --- .../server/blockmanagement/BlockManager.java | 27 ++++++++-- .../blockmanagement/TestBlockManager.java | 54 +++++++++++++++++++ .../hdfs/server/namenode/TestMetaSave.java | 2 + 3 files changed, 79 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index fc754a0c22..6129db8a5a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -705,17 +705,36 @@ public void metaSave(PrintWriter out) { datanodeManager.fetchDatanodes(live, dead, false); out.println("Live Datanodes: " + live.size()); out.println("Dead Datanodes: " + dead.size()); + // - // Dump contents of neededReconstruction + // Need to iterate over all queues from neededReplications + // except for the QUEUE_WITH_CORRUPT_BLOCKS) // synchronized (neededReconstruction) { out.println("Metasave: Blocks waiting for reconstruction: " - + neededReconstruction.size()); - for (Block block : neededReconstruction) { + + neededReconstruction.getLowRedundancyBlockCount()); + for (int i = 0; i < neededReconstruction.LEVEL; i++) { + if (i != neededReconstruction.QUEUE_WITH_CORRUPT_BLOCKS) { + for (Iterator it = neededReconstruction.iterator(i); + it.hasNext();) { + Block block = it.next(); + dumpBlockMeta(block, out); + } + } + } + // + // Now prints corrupt blocks separately + // + out.println("Metasave: Blocks currently missing: " + + neededReconstruction.getCorruptBlockSize()); + for (Iterator it = neededReconstruction. + iterator(neededReconstruction.QUEUE_WITH_CORRUPT_BLOCKS); + it.hasNext();) { + Block block = it.next(); dumpBlockMeta(block, out); } } - + // Dump any postponed over-replicated blocks out.println("Mis-replicated blocks that have been postponed:"); for (Block block : postponedMisreplicatedBlocks) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java index 6b1a9795de..42aeadf272 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java @@ -1459,4 +1459,58 @@ private void verifyPlacementPolicy(final MiniDFSCluster cluster, } } + @Test + public void testMetaSaveMissingReplicas() throws Exception { + List origStorages = getStorages(0, 1); + List origNodes = getNodes(origStorages); + BlockInfo block = makeBlockReplicasMissing(0, origNodes); + File file = new File("test.log"); + PrintWriter out = new PrintWriter(file); + bm.metaSave(out); + out.flush(); + FileInputStream fstream = new FileInputStream(file); + DataInputStream in = new DataInputStream(fstream); + BufferedReader reader = new BufferedReader(new InputStreamReader(in)); + StringBuffer buffer = new StringBuffer(); + String line; + try { + while ((line = reader.readLine()) != null) { + buffer.append(line); + } + String output = buffer.toString(); + assertTrue("Metasave output should have reported missing blocks.", + output.contains("Metasave: Blocks currently missing: 1")); + assertTrue("There should be 0 blocks waiting for reconstruction", + output.contains("Metasave: Blocks waiting for reconstruction: 0")); + String blockNameGS = block.getBlockName() + "_" + + block.getGenerationStamp(); + assertTrue("Block " + blockNameGS + " should be MISSING.", + output.contains(blockNameGS + " MISSING")); + } finally { + reader.close(); + file.delete(); + } + } + + private BlockInfo makeBlockReplicasMissing(long blockId, + List nodesList) throws IOException { + long inodeId = ++mockINodeId; + final INodeFile bc = TestINodeFile.createINodeFile(inodeId); + + BlockInfo blockInfo = blockOnNodes(blockId, nodesList); + blockInfo.setReplication((short) 3); + blockInfo.setBlockCollectionId(inodeId); + + Mockito.doReturn(bc).when(fsn).getBlockCollection(inodeId); + bm.blocksMap.addBlockCollection(blockInfo, bc); + bm.markBlockReplicasAsCorrupt(blockInfo, blockInfo, + blockInfo.getGenerationStamp() + 1, + blockInfo.getNumBytes(), + new DatanodeStorageInfo[]{}); + BlockCollection mockedBc = Mockito.mock(BlockCollection.class); + Mockito.when(mockedBc.getBlocks()).thenReturn(new BlockInfo[]{blockInfo}); + bm.checkRedundancy(mockedBc); + return blockInfo; + } + } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java index 0303a5d8d7..8cc1433cef 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestMetaSave.java @@ -155,6 +155,8 @@ public void testMetasaveAfterDelete() line = reader.readLine(); assertTrue(line.equals("Metasave: Blocks waiting for reconstruction: 0")); line = reader.readLine(); + assertTrue(line.equals("Metasave: Blocks currently missing: 0")); + line = reader.readLine(); assertTrue(line.equals("Mis-replicated blocks that have been postponed:")); line = reader.readLine(); assertTrue(line.equals("Metasave: Blocks being reconstructed: 0"));