From 793820823325e390bb671f4cc1b3bc6920bba5de Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Mon, 9 Oct 2017 13:03:32 -0700 Subject: [PATCH] HDFS-12541. Extend TestSafeModeWithStripedFile with a random EC policy. Contributed by Takanobu Asanuma. --- .../hdfs/TestSafeModeWithStripedFile.java | 29 +++++++---- ...ModeWithStripedFileWithRandomECPolicy.java | 49 +++++++++++++++++++ 2 files changed, 68 insertions(+), 10 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFileWithRandomECPolicy.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java index 3d3ec9c6c6..a43cc52a69 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java @@ -42,13 +42,12 @@ public class TestSafeModeWithStripedFile { - private final ErasureCodingPolicy ecPolicy = - StripedFileTestUtil.getDefaultECPolicy(); - private final short dataBlocks = (short) ecPolicy.getNumDataUnits(); - private final short parityBlocks = (short) ecPolicy.getNumParityUnits(); - private final int numDNs = dataBlocks + parityBlocks; - private final int cellSize = ecPolicy.getCellSize(); - private final int blockSize = cellSize * 2; + private ErasureCodingPolicy ecPolicy; + private short dataBlocks; + private short parityBlocks; + private int numDNs; + private int cellSize; + private int blockSize; private MiniDFSCluster cluster; private Configuration conf; @@ -56,17 +55,27 @@ public class TestSafeModeWithStripedFile { @Rule public Timeout globalTimeout = new Timeout(300000); + public ErasureCodingPolicy getEcPolicy() { + return StripedFileTestUtil.getDefaultECPolicy(); + } + @Before public void setup() throws IOException { + ecPolicy = getEcPolicy(); + dataBlocks = (short) ecPolicy.getNumDataUnits(); + parityBlocks = (short) ecPolicy.getNumParityUnits(); + numDNs = dataBlocks + parityBlocks; + cellSize = ecPolicy.getCellSize(); + blockSize = cellSize * 2; + conf = new HdfsConfiguration(); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 100); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); + cluster.getFileSystem().enableErasureCodingPolicy(getEcPolicy().getName()); cluster.getFileSystem().getClient().setErasureCodingPolicy("/", - StripedFileTestUtil.getDefaultECPolicy().getName()); + getEcPolicy().getName()); cluster.waitActive(); - cluster.getFileSystem().enableErasureCodingPolicy( - StripedFileTestUtil.getDefaultECPolicy().getName()); } @After diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFileWithRandomECPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFileWithRandomECPolicy.java new file mode 100644 index 0000000000..00c29dc42d --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFileWithRandomECPolicy.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This test extends TestSafeModeWithStripedFile to use a random + * (non-default) EC policy. + */ +public class TestSafeModeWithStripedFileWithRandomECPolicy extends + TestSafeModeWithStripedFile { + private static final Logger LOG = LoggerFactory.getLogger( + TestSafeModeWithStripedFileWithRandomECPolicy.class); + + private ErasureCodingPolicy ecPolicy; + + public TestSafeModeWithStripedFileWithRandomECPolicy() { + // If you want to debug this test with a specific ec policy, please use + // SystemErasureCodingPolicies class. + // e.g. ecPolicy = SystemErasureCodingPolicies.getByID(RS_3_2_POLICY_ID); + ecPolicy = StripedFileTestUtil.getRandomNonDefaultECPolicy(); + LOG.info("run {} with {}.", + TestSafeModeWithStripedFileWithRandomECPolicy.class + .getSuperclass().getSimpleName(), ecPolicy.getName()); + } + + @Override + public ErasureCodingPolicy getEcPolicy() { + return ecPolicy; + } +}