From 3b8dbc2cb766ba9fc1d655c891d32f5b4e4aa9c8 Mon Sep 17 00:00:00 2001 From: Xiao Chen Date: Tue, 13 Mar 2018 10:12:52 -0700 Subject: [PATCH] HDFS-12587. Use Parameterized tests in TestBlockInfoStriped and TestLowRedundancyBlockQueues to test all EC policies. Contributed by Takanobu Asanuma. --- .../hadoop/hdfs/StripedFileTestUtil.java | 15 +++++++++ .../blockmanagement/TestBlockInfoStriped.java | 33 +++++++++++++------ .../TestLowRedundancyBlockQueues.java | 19 +++++++++-- 3 files changed, 55 insertions(+), 12 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java index 13ca39026d..35edab93cc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java @@ -46,6 +46,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -591,4 +592,18 @@ public static ErasureCodingPolicy getRandomNonDefaultECPolicy() { .getPolicies(); return policies.get(1 + rand.nextInt(policies.size() - 1)); } + + /** + * Get all Erasure Coding Policies for Parameterized tests. + * @return Collection + */ + public static Collection getECPolicies() { + ArrayList params = new ArrayList<>(); + List policies = + SystemErasureCodingPolicies.getPolicies(); + for (ErasureCodingPolicy policy: policies) { + params.add(new Object[]{policy}); + } + return params; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java index 1040d21857..becf868333 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java @@ -25,29 +25,42 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.Timeout; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import org.mockito.internal.util.reflection.Whitebox; import java.io.DataOutput; import java.io.DataOutputStream; import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; +import java.util.Collection; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; /** - * Test {@link BlockInfoStriped} + * Test {@link BlockInfoStriped}. */ +@RunWith(Parameterized.class) public class TestBlockInfoStriped { private static final long BASE_ID = -1600; private final Block baseBlock = new Block(BASE_ID); - private final ErasureCodingPolicy testECPolicy - = StripedFileTestUtil.getDefaultECPolicy(); - private final int totalBlocks = testECPolicy.getNumDataUnits() + - testECPolicy.getNumParityUnits(); - private final BlockInfoStriped info = new BlockInfoStriped(baseBlock, - testECPolicy); + private final ErasureCodingPolicy testECPolicy; + private final int totalBlocks; + private final BlockInfoStriped info; + + public TestBlockInfoStriped(ErasureCodingPolicy policy) { + testECPolicy = policy; + totalBlocks = testECPolicy.getNumDataUnits() + + testECPolicy.getNumParityUnits(); + info = new BlockInfoStriped(baseBlock, testECPolicy); + } + + @Parameterized.Parameters(name = "{index}: {0}") + public static Collection policies() { + return StripedFileTestUtil.getECPolicies(); + } private Block[] createReportedBlocks(int num) { Block[] blocks = new Block[num]; @@ -61,7 +74,7 @@ private Block[] createReportedBlocks(int num) { public Timeout globalTimeout = new Timeout(300000); /** - * Test adding storage and reported block + * Test adding storage and reported block. */ @Test public void testAddStorage() { @@ -108,8 +121,8 @@ public void testAddStorage() { } // the same block is reported from another storage - DatanodeStorageInfo[] storageInfos2 = DFSTestUtil.createDatanodeStorageInfos( - totalBlocks * 2); + DatanodeStorageInfo[] storageInfos2 = + DFSTestUtil.createDatanodeStorageInfos(totalBlocks * 2); // only add the second half of info2 for (i = totalBlocks; i < storageInfos2.length; i++) { info.addStorage(storageInfos2[i], blocks[i % totalBlocks]); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestLowRedundancyBlockQueues.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestLowRedundancyBlockQueues.java index 0681a0b640..97a5a6e222 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestLowRedundancyBlockQueues.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestLowRedundancyBlockQueues.java @@ -18,22 +18,37 @@ package org.apache.hadoop.hdfs.server.blockmanagement; +import java.util.Collection; import java.util.Iterator; import org.apache.hadoop.hdfs.StripedFileTestUtil; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertFalse; import static org.junit.Assert.fail; +/** + * Test {@link LowRedundancyBlocks}. + */ +@RunWith(Parameterized.class) public class TestLowRedundancyBlockQueues { - private final ErasureCodingPolicy ecPolicy = - StripedFileTestUtil.getDefaultECPolicy(); + private final ErasureCodingPolicy ecPolicy; + + public TestLowRedundancyBlockQueues(ErasureCodingPolicy policy) { + ecPolicy = policy; + } + + @Parameterized.Parameters(name = "{index}: {0}") + public static Collection policies() { + return StripedFileTestUtil.getECPolicies(); + } private BlockInfo genBlockInfo(long id) { return new BlockInfoContiguous(new Block(id), (short) 3);