big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-8666. Speedup the TestMover test. Contributed by Walter Su.

Browse Source
This commit is contained in:
Jing Zhao 2015-07-01 14:59:50 -07:00
parent b5cdf78e8e
commit 152e5df3b6
2 changed files with 21 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -694,6 +694,8 @@ Release 2.8.0 - UNRELEASED
HDFS-8635. Migrate HDFS native build to new CMake framework (Alan Burlison HDFS-8635. Migrate HDFS native build to new CMake framework (Alan Burlison
via Colin P. McCabe) via Colin P. McCabe)
HDFS-8666. Speedup the TestMover tests. (Walter Su via jing9)
OPTIMIZATIONS OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than

19
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
View File

@ -38,6 +38,7 @@
import org.apache.hadoop.hdfs.server.balancer.Dispatcher.DBlock; import org.apache.hadoop.hdfs.server.balancer.Dispatcher.DBlock;
import org.apache.hadoop.hdfs.server.balancer.ExitStatus; import org.apache.hadoop.hdfs.server.balancer.ExitStatus;
import org.apache.hadoop.hdfs.server.balancer.NameNodeConnector; import org.apache.hadoop.hdfs.server.balancer.NameNodeConnector;
import org.apache.hadoop.hdfs.server.balancer.TestBalancer;
import org.apache.hadoop.hdfs.server.mover.Mover.MLocation; import org.apache.hadoop.hdfs.server.mover.Mover.MLocation;
import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.GenericTestUtils;
@ -46,6 +47,21 @@
import org.junit.Test; import org.junit.Test;
public class TestMover { public class TestMover {
static final int DEFAULT_BLOCK_SIZE = 100;
static {
TestBalancer.initTestSetup();
}
static void initConf(Configuration conf) {
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_BLOCK_SIZE);
conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, DEFAULT_BLOCK_SIZE);
conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1L);
conf.setLong(DFSConfigKeys.DFS_BALANCER_MOVEDWINWIDTH_KEY, 2000L);
}
static Mover newMover(Configuration conf) throws IOException { static Mover newMover(Configuration conf) throws IOException {
final Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(conf); final Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(conf);
Assert.assertEquals(1, namenodes.size()); Assert.assertEquals(1, namenodes.size());
@ -97,6 +113,7 @@ public void testScheduleSameBlock() throws IOException {
@Test @Test
public void testScheduleBlockWithinSameNode() throws Exception { public void testScheduleBlockWithinSameNode() throws Exception {
final Configuration conf = new HdfsConfiguration(); final Configuration conf = new HdfsConfiguration();
initConf(conf);
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(3) .numDataNodes(3)
.storageTypes( .storageTypes(
@ -285,6 +302,7 @@ public void testMoverCliWithFederationHA() throws Exception {
public void testTwoReplicaSameStorageTypeShouldNotSelect() throws Exception { public void testTwoReplicaSameStorageTypeShouldNotSelect() throws Exception {
// HDFS-8147 // HDFS-8147
final Configuration conf = new HdfsConfiguration(); final Configuration conf = new HdfsConfiguration();
initConf(conf);
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(3) .numDataNodes(3)
.storageTypes( .storageTypes(
@ -361,6 +379,7 @@ public void testMoveWhenStoragePolicyNotSatisfying() throws Exception {
public void testMoverFailedRetry() throws Exception { public void testMoverFailedRetry() throws Exception {
// HDFS-8147 // HDFS-8147
final Configuration conf = new HdfsConfiguration(); final Configuration conf = new HdfsConfiguration();
initConf(conf);
conf.set(DFSConfigKeys.DFS_MOVER_RETRY_MAX_ATTEMPTS_KEY, "2"); conf.set(DFSConfigKeys.DFS_MOVER_RETRY_MAX_ATTEMPTS_KEY, "2");
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(3) .numDataNodes(3)