diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index f34ca747f3..f4447213c6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -453,6 +453,9 @@ Release 2.7.0 - UNRELEASED HDFS-7406. SimpleHttpProxyHandler puts incorrect "Connection: Close" header. (wheat9) + HDFS-7225. Remove stale block invalidation work when DN re-registers with + different UUID. (Zhe Zhang and Andrew Wang) + Release 2.6.0 - 2014-11-18 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index b8dcd88673..574abccc67 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -1111,6 +1111,18 @@ private void addToInvalidates(Block b) { } } + /** + * Remove all block invalidation tasks under this datanode UUID; + * used when a datanode registers with a new UUID and the old one + * is wiped. + */ + void removeFromInvalidates(final DatanodeInfo datanode) { + if (!namesystem.isPopulatingReplQueues()) { + return; + } + invalidateBlocks.remove(datanode); + } + /** * Mark the block belonging to datanode as corrupt * @param blk Block to be marked as corrupt @@ -3382,7 +3394,14 @@ private int invalidateWorkForOneNode(DatanodeInfo dn) { return 0; } try { - toInvalidate = invalidateBlocks.invalidateWork(datanodeManager.getDatanode(dn)); + DatanodeDescriptor dnDescriptor = datanodeManager.getDatanode(dn); + if (dnDescriptor == null) { + LOG.warn("DataNode " + dn + " cannot be found with UUID " + + dn.getDatanodeUuid() + ", removing block invalidation work."); + invalidateBlocks.remove(dn); + return 0; + } + toInvalidate = invalidateBlocks.invalidateWork(dnDescriptor); if (toInvalidate == null) { return 0; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index d19aad73d4..0780ba3a51 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -600,6 +600,8 @@ private void wipeDatanode(final DatanodeID node) { synchronized (datanodeMap) { host2DatanodeMap.remove(datanodeMap.remove(key)); } + // Also remove all block invalidation tasks under this node + blockManager.removeFromInvalidates(new DatanodeInfo(node)); if (LOG.isDebugEnabled()) { LOG.debug(getClass().getSimpleName() + ".wipeDatanode(" + node + "): storage " + key diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java index d0edd48156..fecca4e915 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestComputeInvalidateWork.java @@ -17,66 +17,161 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; +import java.util.UUID; + import static org.junit.Assert.assertEquals; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.DatanodeID; +import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; import org.apache.hadoop.hdfs.server.common.GenerationStamp; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; +import org.apache.hadoop.hdfs.server.common.StorageInfo; +import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; +import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; +import org.apache.hadoop.util.VersionInfo; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import org.mockito.internal.util.reflection.Whitebox; /** * Test if FSNamesystem handles heartbeat right */ public class TestComputeInvalidateWork { - /** - * Test if {@link FSNamesystem#computeInvalidateWork(int)} - * can schedule invalidate work correctly - */ - @Test - public void testCompInvalidate() throws Exception { - final Configuration conf = new HdfsConfiguration(); - final int NUM_OF_DATANODES = 3; - final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_OF_DATANODES).build(); - try { - cluster.waitActive(); - final FSNamesystem namesystem = cluster.getNamesystem(); - final BlockManager bm = namesystem.getBlockManager(); - final int blockInvalidateLimit = bm.getDatanodeManager().blockInvalidateLimit; - final DatanodeDescriptor[] nodes = bm.getDatanodeManager( - ).getHeartbeatManager().getDatanodes(); - assertEquals(nodes.length, NUM_OF_DATANODES); - - namesystem.writeLock(); - try { - for (int i=0; i