big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Projects Wiki Activity

HDFS-6828. Separate block replica dispatching from Balancer. Contributed by Tsz Wo Nicholas Sze.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1616889 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jing Zhao 2014-08-08 21:33:57 +00:00
parent 05d1bf4157
commit c3cf331dc9
8 changed files with 1111 additions and 1016 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -384,6 +384,9 @@ Release 2.6.0 - UNRELEASED
HDFS-573. Porting libhdfs to Windows. (cnauroth) HDFS-573. Porting libhdfs to Windows. (cnauroth)
HDFS-6828. Separate block replica dispatching from Balancer. (szetszwo via
jing9)
OPTIMIZATIONS OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang) HDFS-6690. Deduplicate xattr names in memory. (wang)

1034
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
View File

File diff suppressed because it is too large Load Diff

1060
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java Normal file
View File

File diff suppressed because it is too large Load Diff

18
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
View File

@ -34,6 +34,10 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.NameNodeProxies; import org.apache.hadoop.hdfs.NameNodeProxies;
import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException; import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.IOUtils;
@ -90,14 +94,16 @@ public class NameNodeConnector implements Closeable {
return blockpoolID; return blockpoolID;
} }
/** @return the namenode proxy. */ /** @return blocks with locations. */
public NamenodeProtocol getNamenode() { public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size)
return namenode; throws IOException {
return namenode.getBlocks(datanode, size);
} }
/** @return the client proxy. */ /** @return live datanode storage reports. */
public ClientProtocol getClient() { public DatanodeStorageReport[] getLiveDatanodeStorageReport()
return client; throws IOException {
return client.getDatanodeStorageReport(DatanodeReportType.LIVE);
} }
/** @return the key manager */ /** @return the key manager */

6
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
View File

@ -89,7 +89,7 @@ public class TestBalancer {
private static final Random r = new Random(); private static final Random r = new Random();
static { static {
Balancer.setBlockMoveWaitTime(1000L) ; Dispatcher.setBlockMoveWaitTime(1000L) ;
} }
static void initConf(Configuration conf) { static void initConf(Configuration conf) {
@ -305,12 +305,12 @@ public class TestBalancer {
for (DatanodeInfo datanode : datanodeReport) { for (DatanodeInfo datanode : datanodeReport) {
double nodeUtilization = ((double)datanode.getDfsUsed()) double nodeUtilization = ((double)datanode.getDfsUsed())
/ datanode.getCapacity(); / datanode.getCapacity();
if (Balancer.Util.shouldBeExcluded(p.nodesToBeExcluded, datanode)) { if (Dispatcher.Util.isExcluded(p.nodesToBeExcluded, datanode)) {
assertTrue(nodeUtilization == 0); assertTrue(nodeUtilization == 0);
actualExcludedNodeCount++; actualExcludedNodeCount++;
continue; continue;
} }
if (!Balancer.Util.shouldBeIncluded(p.nodesToBeIncluded, datanode)) { if (!Dispatcher.Util.isIncluded(p.nodesToBeIncluded, datanode)) {
assertTrue(nodeUtilization == 0); assertTrue(nodeUtilization == 0);
actualExcludedNodeCount++; actualExcludedNodeCount++;
continue; continue;

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java
View File

@ -44,7 +44,7 @@ public class TestBalancerWithHANameNodes {
ClientProtocol client; ClientProtocol client;
static { static {
Balancer.setBlockMoveWaitTime(1000L); Dispatcher.setBlockMoveWaitTime(1000L);
} }
/** /**

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java
View File

@ -73,7 +73,7 @@ public class TestBalancerWithMultipleNameNodes {
private static final Random RANDOM = new Random(); private static final Random RANDOM = new Random();
static { static {
Balancer.setBlockMoveWaitTime(1000L) ; Dispatcher.setBlockMoveWaitTime(1000L) ;
} }
/** Common objects used in various methods. */ /** Common objects used in various methods. */

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java
View File

@ -75,7 +75,7 @@ public class TestBalancerWithNodeGroup {
static final int DEFAULT_BLOCK_SIZE = 100; static final int DEFAULT_BLOCK_SIZE = 100;
static { static {
Balancer.setBlockMoveWaitTime(1000L) ; Dispatcher.setBlockMoveWaitTime(1000L) ;
} }
static Configuration createConf() { static Configuration createConf() {