From 5b0187dcd3dee773f58791ff75213bf71527fdb3 Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Wed, 16 Jan 2013 04:22:38 +0000 Subject: [PATCH] Revert r1433755: HDFS-4288. NN accepts incremental BR as IBR in safemode. Contributed by Daryn Sharp. This commit caused TestBlockManager to fail. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1433820 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 - .../server/blockmanagement/BlockManager.java | 6 +- .../blockmanagement/DatanodeDescriptor.java | 1 - .../blockmanagement/DatanodeManager.java | 2 +- .../blockmanagement/TestBlockManager.java | 69 ------------------- 5 files changed, 2 insertions(+), 79 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 1066112851..3c02599776 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -716,9 +716,6 @@ Release 2.0.3-alpha - Unreleased HDFS-1245. Pluggable block id generation. (shv) - HDFS-4288. NN accepts incremental BR as IBR in safemode - (Daryn Sharp via todd) - BREAKDOWN OF HDFS-3077 SUBTASKS HDFS-3077. Quorum-based protocol for reading and writing edit logs. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 2d08bc4b69..c56f353db3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -62,7 +62,6 @@ import org.apache.hadoop.hdfs.server.namenode.FSClusterStats; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.Namesystem; -import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics; import org.apache.hadoop.hdfs.server.protocol.BlockCommand; import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations; import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations; @@ -1577,10 +1576,7 @@ public void processReport(final DatanodeID nodeID, final String poolId, } // Log the block report processing stats from Namenode perspective - final NameNodeMetrics metrics = NameNode.getNameNodeMetrics(); - if (metrics != null) { - metrics.addBlockReport((int) (endTime - startTime)); - } + NameNode.getNameNodeMetrics().addBlockReport((int) (endTime - startTime)); blockLog.info("BLOCK* processReport: from " + nodeID + ", blocks: " + newReport.getNumberOfBlocks() + ", processing time: " + (endTime - startTime) + " msecs"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java index c542ae343e..a0f445a93b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java @@ -547,7 +547,6 @@ public int getVolumeFailures() { @Override public void updateRegInfo(DatanodeID nodeReg) { super.updateRegInfo(nodeReg); - firstBlockReport = true; // must re-process IBR after re-registration } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index e0a81e0d40..93bf5ff0ab 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -419,7 +419,7 @@ boolean isDatanodeDead(DatanodeDescriptor node) { } /** Add a datanode. */ - void addDatanode(final DatanodeDescriptor node) { + private void addDatanode(final DatanodeDescriptor node) { // To keep host2DatanodeMap consistent with datanodeMap, // remove from host2DatanodeMap the datanodeDescriptor removed // from datanodeMap before adding node to host2DatanodeMap. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java index eabbda064e..b9811e7e5e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java @@ -34,16 +34,13 @@ import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.protocol.Block; -import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; -import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; import org.apache.hadoop.net.NetworkTopology; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; -import static org.mockito.Mockito.*; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; @@ -488,70 +485,4 @@ public void testHighestPriReplSrcChosenDespiteMaxReplLimit() throws Exception { new NumberReplicas(), UnderReplicatedBlocks.QUEUE_HIGHEST_PRIORITY)); } - - @Test - public void testSafeModeIBR() throws Exception { - DatanodeDescriptor node = spy(nodes.get(0)); - node.setStorageID("dummy-storage"); - node.isAlive = true; - - DatanodeRegistration nodeReg = - new DatanodeRegistration(node, null, null, ""); - - // pretend to be in safemode - doReturn(true).when(fsn).isInStartupSafeMode(); - - // register new node - bm.getDatanodeManager().registerDatanode(nodeReg); - bm.getDatanodeManager().addDatanode(node); // swap in spy - assertEquals(node, bm.getDatanodeManager().getDatanode(node)); - assertTrue(node.isFirstBlockReport()); - // send block report, should be processed - reset(node); - bm.processReport(node, "pool", new BlockListAsLongs()); - verify(node).receivedBlockReport(); - assertFalse(node.isFirstBlockReport()); - // send block report again, should NOT be processed - reset(node); - bm.processReport(node, "pool", new BlockListAsLongs()); - verify(node, never()).receivedBlockReport(); - assertFalse(node.isFirstBlockReport()); - - // re-register as if node restarted, should update existing node - bm.getDatanodeManager().removeDatanode(node); - reset(node); - bm.getDatanodeManager().registerDatanode(nodeReg); - verify(node).updateRegInfo(nodeReg); - assertTrue(node.isFirstBlockReport()); // ready for report again - // send block report, should be processed after restart - reset(node); - bm.processReport(node, "pool", new BlockListAsLongs()); - verify(node).receivedBlockReport(); - assertFalse(node.isFirstBlockReport()); - } - - @Test - public void testSafeModeIBRAfterIncremental() throws Exception { - DatanodeDescriptor node = spy(nodes.get(0)); - node.setStorageID("dummy-storage"); - node.isAlive = true; - - DatanodeRegistration nodeReg = - new DatanodeRegistration(node, null, null, ""); - - // pretend to be in safemode - doReturn(true).when(fsn).isInStartupSafeMode(); - - // register new node - bm.getDatanodeManager().registerDatanode(nodeReg); - bm.getDatanodeManager().addDatanode(node); // swap in spy - assertEquals(node, bm.getDatanodeManager().getDatanode(node)); - assertTrue(node.isFirstBlockReport()); - // send block report while pretending to already have blocks - reset(node); - doReturn(1).when(node).numBlocks(); - bm.processReport(node, "pool", new BlockListAsLongs()); - verify(node).receivedBlockReport(); - assertFalse(node.isFirstBlockReport()); - } }