From 75cdb5bb4965161021df47376cccf058bf413f3b Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Mon, 22 Oct 2012 18:30:16 +0000 Subject: [PATCH] HDFS-4099. Clean up replication code and add more javadoc. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1400986 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../server/blockmanagement/BlockManager.java | 59 ++++++++++--------- .../hdfs/server/namenode/FSNamesystem.java | 17 +----- 3 files changed, 34 insertions(+), 44 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 0d19eebdf6..8acc75bd91 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -408,6 +408,8 @@ Release 2.0.3-alpha - Unreleased HDFS-4088. Remove "throws QuotaExceededException" from an INodeDirectoryWithQuota constructor. (szetszwo) + HDFS-4099. Clean up replication code and add more javadoc. (szetszwo) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 81ec0b5e2e..e7aff0d61e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; +import static org.apache.hadoop.util.ExitUtil.terminate; + import java.io.IOException; import java.io.PrintWriter; import java.util.ArrayList; @@ -49,14 +51,11 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException; -import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey; import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager; import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager.AccessMode; +import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey; import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; import org.apache.hadoop.hdfs.server.blockmanagement.PendingDataNodeMessages.ReportedBlockInfo; - -import static org.apache.hadoop.util.ExitUtil.terminate; - import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState; import org.apache.hadoop.hdfs.server.namenode.FSClusterStats; @@ -2833,28 +2832,32 @@ private void updateNeededReplications(final Block block, } } - public void checkReplication(Block block, short numExpectedReplicas) { - // filter out containingNodes that are marked for decommission. - NumberReplicas number = countNodes(block); - if (isNeededReplication(block, numExpectedReplicas, number.liveReplicas())) { - neededReplications.add(block, - number.liveReplicas(), - number.decommissionedReplicas(), - numExpectedReplicas); - return; - } - if (number.liveReplicas() > numExpectedReplicas) { - processOverReplicatedBlock(block, numExpectedReplicas, null, null); + /** + * Check replication of the blocks in the collection. + * If any block is needed replication, insert it into the replication queue. + * Otherwise, if the block is more than the expected replication factor, + * process it as an over replicated block. + */ + public void checkReplication(BlockCollection bc) { + final short expected = bc.getBlockReplication(); + for (Block block : bc.getBlocks()) { + final NumberReplicas n = countNodes(block); + if (isNeededReplication(block, expected, n.liveReplicas())) { + neededReplications.add(block, n.liveReplicas(), + n.decommissionedReplicas(), expected); + } else if (n.liveReplicas() > expected) { + processOverReplicatedBlock(block, expected, null, null); + } } } - /* get replication factor of a block */ + /** + * @return 0 if the block is not found; + * otherwise, return the replication factor of the block. + */ private int getReplication(Block block) { - BlockCollection bc = blocksMap.getBlockCollection(block); - if (bc == null) { // block does not belong to any file - return 0; - } - return bc.getBlockReplication(); + final BlockCollection bc = blocksMap.getBlockCollection(block); + return bc == null? 0: bc.getBlockReplication(); } @@ -2929,12 +2932,12 @@ boolean blockHasEnoughRacks(Block b) { return enoughRacks; } - boolean isNeededReplication(Block b, int expectedReplication, int curReplicas) { - if ((curReplicas >= expectedReplication) && (blockHasEnoughRacks(b))) { - return false; - } else { - return true; - } + /** + * A block needs replication if the number of replicas is less than expected + * or if it does not have enough racks. + */ + private boolean isNeededReplication(Block b, int expected, int current) { + return current < expected || !blockHasEnoughRacks(b); } public long getMissingBlocksCount() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 0cbbd1a8c1..fcd21da2fb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -2428,21 +2428,6 @@ private boolean completeFileInternal(String src, return true; } - /** - * Check all blocks of a file. If any blocks are lower than their intended - * replication factor, then insert them into neededReplication and if - * the blocks are more than the intended replication factor then insert - * them into invalidateBlocks. - */ - private void checkReplicationFactor(INodeFile file) { - short numExpectedReplicas = file.getBlockReplication(); - Block[] pendingBlocks = file.getBlocks(); - int nrBlocks = pendingBlocks.length; - for (int i = 0; i < nrBlocks; i++) { - blockManager.checkReplication(pendingBlocks[i], numExpectedReplicas); - } - } - /** * Allocate a block at the given pending filename * @@ -3175,7 +3160,7 @@ private void finalizeINodeFileUnderConstruction(String src, // close file and persist block allocations for this file dir.closeFile(src, newFile); - checkReplicationFactor(newFile); + blockManager.checkReplication(newFile); } void commitBlockSynchronization(ExtendedBlock lastblock,