big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-11916. Extend TestErasureCodingPolicies/TestErasureCodingPolicyWithSnapshot with a random EC policy. Contributed by Takanobu Asanuma.

Browse Source
This commit is contained in:
Lei Xu 2017-06-19 10:25:20 -07:00
parent ee89ac84e6
commit 73fb75017e
4 changed files with 155 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
View File

@ -61,15 +61,19 @@ public class TestErasureCodingPolicies {
private MiniDFSCluster cluster;
private DistributedFileSystem fs;
private static final int BLOCK_SIZE = 1024;
private static final ErasureCodingPolicy EC_POLICY =
StripedFileTestUtil.getDefaultECPolicy();
private ErasureCodingPolicy ecPolicy;
private FSNamesystem namesystem;
public ErasureCodingPolicy getEcPolicy() {
return StripedFileTestUtil.getDefaultECPolicy();
}
@Rule
public Timeout timeout = new Timeout(60 * 1000);
@Before
public void setupCluster() throws IOException {
ecPolicy = getEcPolicy();
conf = new HdfsConfiguration();
conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
DFSTestUtil.enableAllECPolicies(conf);
@ -100,8 +104,7 @@ public void testReplicatedFileUnderECDir() throws IOException {
DFSTestUtil.createFile(fs, replicatedFile, 0, (short) 3, 0L);
// set ec policy on dir
fs.setErasureCodingPolicy(dir,
StripedFileTestUtil.getDefaultECPolicy().getName());
fs.setErasureCodingPolicy(dir, ecPolicy.getName());
// create a file which should be using ec
final Path ecSubDir = new Path(dir, "ecSubDir");
final Path ecFile = new Path(ecSubDir, "ecFile");
@ -153,7 +156,7 @@ public void testBasicSetECPolicy()
fs.mkdir(testDir, FsPermission.getDirDefault());
/* Normal creation of an erasure coding directory */
fs.setErasureCodingPolicy(testDir, EC_POLICY.getName());
fs.setErasureCodingPolicy(testDir, ecPolicy.getName());
/* Verify files under the directory are striped */
final Path ECFilePath = new Path(testDir, "foo");
@ -169,7 +172,7 @@ public void testBasicSetECPolicy()
fs.mkdir(notEmpty, FsPermission.getDirDefault());
final Path oldFile = new Path(notEmpty, "old");
fs.create(oldFile);
fs.setErasureCodingPolicy(notEmpty, EC_POLICY.getName());
fs.setErasureCodingPolicy(notEmpty, ecPolicy.getName());
final Path newFile = new Path(notEmpty, "new");
fs.create(newFile);
INode oldInode = namesystem.getFSDirectory().getINode(oldFile.toString());
@ -181,10 +184,10 @@ public void testBasicSetECPolicy()
final Path dir1 = new Path("/dir1");
final Path dir2 = new Path(dir1, "dir2");
fs.mkdir(dir1, FsPermission.getDirDefault());
fs.setErasureCodingPolicy(dir1, EC_POLICY.getName());
fs.setErasureCodingPolicy(dir1, ecPolicy.getName());
fs.mkdir(dir2, FsPermission.getDirDefault());
try {
fs.setErasureCodingPolicy(dir2, EC_POLICY.getName());
fs.setErasureCodingPolicy(dir2, ecPolicy.getName());
} catch (IOException e) {
fail("Nested erasure coding policies are supported");
}
@ -193,7 +196,7 @@ public void testBasicSetECPolicy()
final Path fPath = new Path("/file");
fs.create(fPath);
try {
fs.setErasureCodingPolicy(fPath, EC_POLICY.getName());
fs.setErasureCodingPolicy(fPath, ecPolicy.getName());
fail("Erasure coding policy on file");
} catch (IOException e) {
assertExceptionContains("erasure coding policy for a file", e);
@ -213,11 +216,11 @@ public void testBasicSetECPolicy()
// Already set directory-level policies should still be in effect
Path disabledPolicy = new Path(dir1, "afterDisabled");
Assert.assertEquals("Dir does not have policy set",
EC_POLICY,
ecPolicy,
fs.getErasureCodingPolicy(dir1));
fs.create(disabledPolicy).close();
Assert.assertEquals("File did not inherit dir's policy",
EC_POLICY,
ecPolicy,
fs.getErasureCodingPolicy(disabledPolicy));
// Also check loading disabled EC policies from fsimage
@ -227,10 +230,10 @@ public void testBasicSetECPolicy()
cluster.restartNameNodes();
Assert.assertEquals("Dir does not have policy set",
EC_POLICY,
ecPolicy,
fs.getErasureCodingPolicy(dir1));
Assert.assertEquals("File does not have policy set",
EC_POLICY,
ecPolicy,
fs.getErasureCodingPolicy(disabledPolicy));
}
@ -240,8 +243,8 @@ public void testMoveValidity() throws IOException, InterruptedException {
final Path dstECDir = new Path("/dstEC");
fs.mkdir(srcECDir, FsPermission.getDirDefault());
fs.mkdir(dstECDir, FsPermission.getDirDefault());
fs.setErasureCodingPolicy(srcECDir, EC_POLICY.getName());
fs.setErasureCodingPolicy(dstECDir, EC_POLICY.getName());
fs.setErasureCodingPolicy(srcECDir, ecPolicy.getName());
fs.setErasureCodingPolicy(dstECDir, ecPolicy.getName());
final Path srcFile = new Path(srcECDir, "foo");
fs.create(srcFile);
@ -275,8 +278,7 @@ public void testMoveValidity() throws IOException, InterruptedException {
public void testReplication() throws IOException {
final Path testDir = new Path("/ec");
fs.mkdir(testDir, FsPermission.getDirDefault());
fs.setErasureCodingPolicy(testDir,
StripedFileTestUtil.getDefaultECPolicy().getName());
fs.setErasureCodingPolicy(testDir, ecPolicy.getName());
final Path fooFile = new Path(testDir, "foo");
// create ec file with replication=0
fs.create(fooFile, FsPermission.getFileDefault(), true,
@ -330,10 +332,10 @@ public void testGetErasureCodingPolicy() throws Exception {
private void verifyErasureCodingInfo(
String src, ErasureCodingPolicy usingECPolicy) throws IOException {
HdfsFileStatus hdfsFileStatus = fs.getClient().getFileInfo(src);
ErasureCodingPolicy ecPolicy = hdfsFileStatus.getErasureCodingPolicy();
assertNotNull(ecPolicy);
ErasureCodingPolicy actualPolicy = hdfsFileStatus.getErasureCodingPolicy();
assertNotNull(actualPolicy);
assertEquals("Actually used ecPolicy should be equal with target ecPolicy",
usingECPolicy, ecPolicy);
usingECPolicy, actualPolicy);
}
@Test
@ -342,13 +344,13 @@ public void testSetInvalidPolicy()
ECSchema rsSchema = new ECSchema("rs", 4, 2);
String policyName = "RS-4-2-128k";
int cellSize = 128 * 1024;
ErasureCodingPolicy ecPolicy =
ErasureCodingPolicy invalidPolicy =
new ErasureCodingPolicy(policyName, rsSchema, cellSize, (byte) -1);
String src = "/ecDir4-2";
final Path ecDir = new Path(src);
try {
fs.mkdir(ecDir, FsPermission.getDirDefault());
fs.getClient().setErasureCodingPolicy(src, ecPolicy.getName());
fs.getClient().setErasureCodingPolicy(src, invalidPolicy.getName());
fail("HadoopIllegalArgumentException should be thrown for"
+ "setting an invalid erasure coding policy");
} catch (Exception e) {
@ -429,8 +431,7 @@ public HdfsAdmin run() throws Exception {
Path ecfile = new Path(ecdir, "ecfile");
fs.setPermission(new Path("/"), new FsPermission((short)0777));
userfs.mkdirs(ecdir);
final String ecPolicyName = StripedFileTestUtil.getDefaultECPolicy()
.getName();
final String ecPolicyName = ecPolicy.getName();
useradmin.setErasureCodingPolicy(ecdir, ecPolicyName);
assertEquals("Policy not present on dir",
ecPolicyName,
@ -537,12 +538,12 @@ public void testFileLevelECPolicy() throws Exception {
final Path filePath1 = new Path(dirPath, "file1");
fs.mkdirs(dirPath);
fs.setErasureCodingPolicy(dirPath, EC_POLICY.getName());
fs.setErasureCodingPolicy(dirPath, ecPolicy.getName());
// null EC policy name value means inheriting parent directory's policy
fs.createFile(filePath0).build().close();
ErasureCodingPolicy ecPolicyOnFile = fs.getErasureCodingPolicy(filePath0);
assertEquals(EC_POLICY, ecPolicyOnFile);
assertEquals(ecPolicy, ecPolicyOnFile);
// Test illegal EC policy name
final String illegalPolicyName = "RS-DEFAULT-1-2-64k";
@ -560,7 +561,8 @@ public void testFileLevelECPolicy() throws Exception {
final ErasureCodingPolicy ecPolicyOnDir =
SystemErasureCodingPolicies.getByID(
SystemErasureCodingPolicies.RS_3_2_POLICY_ID);
ecPolicyOnFile = EC_POLICY;
ecPolicyOnFile = SystemErasureCodingPolicies.getByID(
SystemErasureCodingPolicies.RS_6_3_POLICY_ID);
fs.setErasureCodingPolicy(dirPath, ecPolicyOnDir.getName());
fs.createFile(filePath0).ecPolicyName(ecPolicyOnFile.getName())
.build().close();
@ -578,11 +580,11 @@ public void testEnforceAsReplicatedFile() throws Exception {
final Path filePath = new Path(dirPath, "file");
fs.mkdirs(dirPath);
fs.setErasureCodingPolicy(dirPath, EC_POLICY.getName());
fs.setErasureCodingPolicy(dirPath, ecPolicy.getName());
final String ecPolicyName = "RS-10-4-64k";
fs.createFile(filePath).build().close();
assertEquals(EC_POLICY, fs.getErasureCodingPolicy(filePath));
assertEquals(ecPolicy, fs.getErasureCodingPolicy(filePath));
fs.delete(filePath, true);
fs.createFile(filePath)

48
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPoliciesWithRandomECPolicy.java Normal file
View File

@ -0,0 +1,48 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* This test extends TestErasureCodingPolicies to use a random (non-default) EC
* policy.
*/
public class TestErasureCodingPoliciesWithRandomECPolicy extends
TestErasureCodingPolicies {
private static final Logger LOG = LoggerFactory.getLogger(
TestErasureCodingPoliciesWithRandomECPolicy.class);
private ErasureCodingPolicy ecPolicy;
public TestErasureCodingPoliciesWithRandomECPolicy() {
// If you want to debug this test with a specific ec policy, please use
// SystemErasureCodingPolicies class.
// e.g. ecPolicy = SystemErasureCodingPolicies.getByID(RS_3_2_POLICY_ID);
ecPolicy = StripedFileTestUtil.getRandomNonDefaultECPolicy();
LOG.info("run {} with {}.", TestErasureCodingPoliciesWithRandomECPolicy
.class.getSuperclass().getSimpleName(), ecPolicy.getName());
}
@Override
public ErasureCodingPolicy getEcPolicy() {
return ecPolicy;
}
}

50
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java
View File

@ -40,17 +40,21 @@ public class TestErasureCodingPolicyWithSnapshot {
private Configuration conf;
private final static int SUCCESS = 0;
private final ErasureCodingPolicy sysDefaultPolicy =
StripedFileTestUtil.getDefaultECPolicy();
private final short groupSize = (short) (
sysDefaultPolicy.getNumDataUnits() +
sysDefaultPolicy.getNumParityUnits());
private ErasureCodingPolicy ecPolicy;
private short groupSize;
public ErasureCodingPolicy getEcPolicy() {
return StripedFileTestUtil.getDefaultECPolicy();
}
@Before
public void setupCluster() throws IOException {
ecPolicy = getEcPolicy();
groupSize = (short) (ecPolicy.getNumDataUnits()
+ ecPolicy.getNumParityUnits());
conf = new HdfsConfiguration();
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
sysDefaultPolicy.getName());
ecPolicy.getName());
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize).build();
cluster.waitActive();
fs = cluster.getFileSystem();
@ -77,12 +81,12 @@ public void testSnapshotsOnErasureCodingDirsParentDir() throws Exception {
fs.mkdirs(ecDir);
fs.allowSnapshot(ecDirParent);
// set erasure coding policy
fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName());
fs.setErasureCodingPolicy(ecDir, ecPolicy.getName());
DFSTestUtil.createFile(fs, ecFile, len, (short) 1, 0xFEED);
String contents = DFSTestUtil.readFile(fs, ecFile);
final Path snap1 = fs.createSnapshot(ecDirParent, "snap1");
final Path snap1ECDir = new Path(snap1, ecDir.getName());
assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
assertEquals("Got unexpected erasure coding policy", ecPolicy,
fs.getErasureCodingPolicy(snap1ECDir));
// Now delete the dir which has erasure coding policy. Re-create the dir again, and
@ -95,18 +99,18 @@ public void testSnapshotsOnErasureCodingDirsParentDir() throws Exception {
fs.getErasureCodingPolicy(snap2ECDir));
// Make dir again with system default ec policy
fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName());
fs.setErasureCodingPolicy(ecDir, ecPolicy.getName());
final Path snap3 = fs.createSnapshot(ecDirParent, "snap3");
final Path snap3ECDir = new Path(snap3, ecDir.getName());
// Check that snap3's ECPolicy has the correct settings
ErasureCodingPolicy ezSnap3 = fs.getErasureCodingPolicy(snap3ECDir);
assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
assertEquals("Got unexpected erasure coding policy", ecPolicy,
ezSnap3);
// Check that older snapshots still have the old ECPolicy settings
assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
assertEquals("Got unexpected erasure coding policy", ecPolicy,
fs.getErasureCodingPolicy(snap1ECDir));
assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
assertEquals("Got unexpected erasure coding policy", ecPolicy,
fs.getErasureCodingPolicy(snap2ECDir));
// Verify contents of the snapshotted file
@ -118,12 +122,12 @@ public void testSnapshotsOnErasureCodingDirsParentDir() throws Exception {
// Now delete the snapshots out of order and verify the EC policy
// correctness
fs.deleteSnapshot(ecDirParent, snap2.getName());
assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
assertEquals("Got unexpected erasure coding policy", ecPolicy,
fs.getErasureCodingPolicy(snap1ECDir));
assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
assertEquals("Got unexpected erasure coding policy", ecPolicy,
fs.getErasureCodingPolicy(snap3ECDir));
fs.deleteSnapshot(ecDirParent, snap1.getName());
assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
assertEquals("Got unexpected erasure coding policy", ecPolicy,
fs.getErasureCodingPolicy(snap3ECDir));
}
@ -136,9 +140,9 @@ public void testSnapshotsOnErasureCodingDir() throws Exception {
fs.mkdirs(ecDir);
fs.allowSnapshot(ecDir);
fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName());
fs.setErasureCodingPolicy(ecDir, ecPolicy.getName());
final Path snap1 = fs.createSnapshot(ecDir, "snap1");
assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
assertEquals("Got unexpected erasure coding policy", ecPolicy,
fs.getErasureCodingPolicy(snap1));
}
@ -152,10 +156,10 @@ public void testSnapshotsOnErasureCodingDirAfterNNRestart() throws Exception {
fs.allowSnapshot(ecDir);
// set erasure coding policy
fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName());
fs.setErasureCodingPolicy(ecDir, ecPolicy.getName());
final Path snap1 = fs.createSnapshot(ecDir, "snap1");
ErasureCodingPolicy ecSnap = fs.getErasureCodingPolicy(snap1);
assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
assertEquals("Got unexpected erasure coding policy", ecPolicy,
ecSnap);
// save namespace, restart namenode, and check ec policy correctness.
@ -165,7 +169,7 @@ public void testSnapshotsOnErasureCodingDirAfterNNRestart() throws Exception {
cluster.restartNameNode(true);
ErasureCodingPolicy ecSnap1 = fs.getErasureCodingPolicy(snap1);
assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
assertEquals("Got unexpected erasure coding policy", ecPolicy,
ecSnap1);
assertEquals("Got unexpected ecSchema", ecSnap.getSchema(),
ecSnap1.getSchema());
@ -184,7 +188,7 @@ public void testCopySnapshotWillNotPreserveErasureCodingPolicy()
fs.allowSnapshot(ecDir);
// set erasure coding policy
fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName());
fs.setErasureCodingPolicy(ecDir, ecPolicy.getName());
DFSTestUtil.createFile(fs, ecFile, len, (short) 1, 0xFEED);
final Path snap1 = fs.createSnapshot(ecDir, "snap1");
@ -197,7 +201,7 @@ public void testCopySnapshotWillNotPreserveErasureCodingPolicy()
assertNull("Got unexpected erasure coding policy",
fs.getErasureCodingPolicy(snap1CopyECDir));
assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
assertEquals("Got unexpected erasure coding policy", ecPolicy,
fs.getErasureCodingPolicy(snap1));
}
@ -212,7 +216,7 @@ public void testFileStatusAcrossNNRestart() throws IOException {
fs.mkdirs(ecDir);
// Set erasure coding policy
fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName());
fs.setErasureCodingPolicy(ecDir, ecPolicy.getName());
DFSTestUtil.createFile(fs, ecFile, len, (short) 1, 0xFEED);
// Verify FileStatus for normal and EC files

49
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshotWithRandomECPolicy.java Normal file
View File

@ -0,0 +1,49 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* This test extends TestErasureCodingPolicyWithSnapshot to use a random
* (non-default) EC policy.
*/
public class TestErasureCodingPolicyWithSnapshotWithRandomECPolicy extends
TestErasureCodingPolicyWithSnapshot {
private static final Logger LOG = LoggerFactory.getLogger(
TestErasureCodingPolicyWithSnapshotWithRandomECPolicy.class);
private ErasureCodingPolicy ecPolicy;
public TestErasureCodingPolicyWithSnapshotWithRandomECPolicy() {
// If you want to debug this test with a specific ec policy, please use
// SystemErasureCodingPolicies class.
// e.g. ecPolicy = SystemErasureCodingPolicies.getByID(RS_3_2_POLICY_ID);
ecPolicy = StripedFileTestUtil.getRandomNonDefaultECPolicy();
LOG.info("run {} with {}.",
TestErasureCodingPolicyWithSnapshotWithRandomECPolicy.class
.getSuperclass().getSimpleName(), ecPolicy.getName());
}
@Override
public ErasureCodingPolicy getEcPolicy() {
return ecPolicy;
}
}