big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-6828. Separate block replica dispatching from Balancer. Contributed by Tsz Wo Nicholas Sze.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1616889 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jing Zhao 2014-08-08 21:33:57 +00:00
parent 05d1bf4157
commit c3cf331dc9
8 changed files with 1111 additions and 1016 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -384,6 +384,9 @@ Release 2.6.0 - UNRELEASED
HDFS-573. Porting libhdfs to Windows. (cnauroth)
HDFS-6828. Separate block replica dispatching from Balancer. (szetszwo via
jing9)
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)

1034
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
View File

File diff suppressed because it is too large Load Diff

1060
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java Normal file
View File

File diff suppressed because it is too large Load Diff

18
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
View File

@ -34,6 +34,10 @@
import org.apache.hadoop.hdfs.NameNodeProxies;
import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.io.IOUtils;
@ -90,14 +94,16 @@ public String getBlockpoolID() {
return blockpoolID;
}
/** @return the namenode proxy. */
public NamenodeProtocol getNamenode() {
return namenode;
/** @return blocks with locations. */
public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size)
throws IOException {
return namenode.getBlocks(datanode, size);
}
/** @return the client proxy. */
public ClientProtocol getClient() {
return client;
/** @return live datanode storage reports. */
public DatanodeStorageReport[] getLiveDatanodeStorageReport()
throws IOException {
return client.getDatanodeStorageReport(DatanodeReportType.LIVE);
}
/** @return the key manager */

6
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
View File

@ -89,7 +89,7 @@ public class TestBalancer {
private static final Random r = new Random();
static {
Balancer.setBlockMoveWaitTime(1000L) ;
Dispatcher.setBlockMoveWaitTime(1000L) ;
}
static void initConf(Configuration conf) {
@ -305,12 +305,12 @@ static void waitForBalancer(long totalUsedSpace, long totalCapacity,
for (DatanodeInfo datanode : datanodeReport) {
double nodeUtilization = ((double)datanode.getDfsUsed())
/ datanode.getCapacity();
if (Balancer.Util.shouldBeExcluded(p.nodesToBeExcluded, datanode)) {
if (Dispatcher.Util.isExcluded(p.nodesToBeExcluded, datanode)) {
assertTrue(nodeUtilization == 0);
actualExcludedNodeCount++;
continue;
}
if (!Balancer.Util.shouldBeIncluded(p.nodesToBeIncluded, datanode)) {
if (!Dispatcher.Util.isIncluded(p.nodesToBeIncluded, datanode)) {
assertTrue(nodeUtilization == 0);
actualExcludedNodeCount++;
continue;

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java
View File

@ -44,7 +44,7 @@ public class TestBalancerWithHANameNodes {
ClientProtocol client;
static {
Balancer.setBlockMoveWaitTime(1000L);
Dispatcher.setBlockMoveWaitTime(1000L);
}
/**

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java
View File

@ -73,7 +73,7 @@ public class TestBalancerWithMultipleNameNodes {
private static final Random RANDOM = new Random();
static {
Balancer.setBlockMoveWaitTime(1000L) ;
Dispatcher.setBlockMoveWaitTime(1000L) ;
}
/** Common objects used in various methods. */

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java
View File

@ -75,7 +75,7 @@ public class TestBalancerWithNodeGroup {
static final int DEFAULT_BLOCK_SIZE = 100;
static {
Balancer.setBlockMoveWaitTime(1000L) ;
Dispatcher.setBlockMoveWaitTime(1000L) ;
}
static Configuration createConf() {