big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-12249. dfsadmin -metaSave to output maintenance mode blocks. Contributed by Wellington Chevreuil.

Browse Source
This commit is contained in:
Wei-Chiu Chuang 2017-10-23 15:23:52 -07:00
parent 9e77dc2bd1
commit 0f2a69127a
2 changed files with 119 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
View File

@ -833,10 +833,12 @@ private void dumpBlockMeta(Block block, PrintWriter out) {
// l: == live:, d: == decommissioned c: == corrupt e: == excess
out.print(block + ((usableReplicas > 0)? "" : " MISSING") +
" (replicas:" +
" l: " + numReplicas.liveReplicas() +
" d: " + numReplicas.decommissionedAndDecommissioning() +
" c: " + numReplicas.corruptReplicas() +
" e: " + numReplicas.excessReplicas() + ") ");
" live: " + numReplicas.liveReplicas() +
" decommissioning and decommissioned: " +
numReplicas.decommissionedAndDecommissioning() +
" corrupt: " + numReplicas.corruptReplicas() +
" in excess: " + numReplicas.excessReplicas() +
" maintenance mode: " + numReplicas.maintenanceReplicas() + ") ");
Collection<DatanodeDescriptor> corruptNodes =
corruptReplicas.getNodes(block);
@ -849,6 +851,8 @@ private void dumpBlockMeta(Block block, PrintWriter out) {
} else if (node.isDecommissioned() ||
node.isDecommissionInProgress()) {
state = "(decommissioned)";
} else if (node.isMaintenance() || node.isInMaintenance()){
state = "(maintenance)";
}
if (storage.areBlockContentsStale()) {

111
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
View File

@ -1511,4 +1511,115 @@ private BlockInfo makeBlockReplicasMissing(long blockId,
return blockInfo;
}
private BlockInfo makeBlockReplicasMaintenance(long blockId,
List<DatanodeDescriptor> nodesList) throws IOException {
long inodeId = ++mockINodeId;
final INodeFile bc = TestINodeFile.createINodeFile(inodeId);
BlockInfo blockInfo = blockOnNodes(blockId, nodesList);
blockInfo.setReplication((short) 3);
blockInfo.setBlockCollectionId(inodeId);
Mockito.doReturn(bc).when(fsn).getBlockCollection(inodeId);
bm.blocksMap.addBlockCollection(blockInfo, bc);
nodesList.get(0).setInMaintenance();
BlockCollection mockedBc = Mockito.mock(BlockCollection.class);
Mockito.when(mockedBc.getBlocks()).thenReturn(new BlockInfo[]{blockInfo});
bm.checkRedundancy(mockedBc);
return blockInfo;
}
@Test
public void testMetaSaveInMaintenanceReplicas() throws Exception {
List<DatanodeStorageInfo> origStorages = getStorages(0, 1);
List<DatanodeDescriptor> origNodes = getNodes(origStorages);
BlockInfo block = makeBlockReplicasMaintenance(0, origNodes);
File file = new File("test.log");
PrintWriter out = new PrintWriter(file);
bm.metaSave(out);
out.flush();
FileInputStream fstream = new FileInputStream(file);
DataInputStream in = new DataInputStream(fstream);
BufferedReader reader = new BufferedReader(new InputStreamReader(in));
StringBuffer buffer = new StringBuffer();
String line;
try {
while ((line = reader.readLine()) != null) {
buffer.append(line);
System.out.println(line);
}
String output = buffer.toString();
assertTrue("Metasave output should not have reported " +
"missing blocks.",
output.contains("Metasave: Blocks currently missing: 0"));
assertTrue("There should be 1 block waiting for reconstruction",
output.contains("Metasave: Blocks waiting for reconstruction: 1"));
String blockNameGS = block.getBlockName() + "_" +
block.getGenerationStamp();
assertTrue("Block " + blockNameGS +
" should be list as maintenance.",
output.contains(blockNameGS + " (replicas: live: 1 decommissioning " +
"and decommissioned: 0 corrupt: 0 in excess: " +
"0 maintenance mode: 1)"));
} finally {
reader.close();
file.delete();
}
}
private BlockInfo makeBlockReplicasDecommission(long blockId,
List<DatanodeDescriptor> nodesList) throws IOException {
long inodeId = ++mockINodeId;
final INodeFile bc = TestINodeFile.createINodeFile(inodeId);
BlockInfo blockInfo = blockOnNodes(blockId, nodesList);
blockInfo.setReplication((short) 3);
blockInfo.setBlockCollectionId(inodeId);
Mockito.doReturn(bc).when(fsn).getBlockCollection(inodeId);
bm.blocksMap.addBlockCollection(blockInfo, bc);
nodesList.get(0).startDecommission();
BlockCollection mockedBc = Mockito.mock(BlockCollection.class);
Mockito.when(mockedBc.getBlocks()).thenReturn(new BlockInfo[]{blockInfo});
bm.checkRedundancy(mockedBc);
return blockInfo;
}
@Test
public void testMetaSaveDecommissioningReplicas() throws Exception {
List<DatanodeStorageInfo> origStorages = getStorages(0, 1);
List<DatanodeDescriptor> origNodes = getNodes(origStorages);
BlockInfo block = makeBlockReplicasDecommission(0, origNodes);
File file = new File("test.log");
PrintWriter out = new PrintWriter(file);
bm.metaSave(out);
out.flush();
FileInputStream fstream = new FileInputStream(file);
DataInputStream in = new DataInputStream(fstream);
BufferedReader reader = new BufferedReader(new InputStreamReader(in));
StringBuffer buffer = new StringBuffer();
String line;
try {
while ((line = reader.readLine()) != null) {
buffer.append(line);
}
String output = buffer.toString();
assertTrue("Metasave output should not have reported " +
"missing blocks.",
output.contains("Metasave: Blocks currently missing: 0"));
assertTrue("There should be 1 block waiting for reconstruction",
output.contains("Metasave: Blocks waiting for reconstruction: 1"));
String blockNameGS = block.getBlockName() + "_" +
block.getGenerationStamp();
assertTrue("Block " + blockNameGS +
" should be list as maintenance.",
output.contains(blockNameGS + " (replicas: live: 1 decommissioning " +
"and decommissioned: 1 corrupt: 0 in excess: " +
"0 maintenance mode: 0)"));
} finally {
reader.close();
file.delete();
}
}
}