From b14db95c35ea17975f3f287f8e4de8aa201d9202 Mon Sep 17 00:00:00 2001 From: Sean Mackrory Date: Tue, 6 Feb 2018 16:33:11 -0700 Subject: [PATCH] HDFS-13106. Need to exercise all HDFS APIs for EC. Contributed by Haibo Yan. --- .../hdfs/TestErasureCodingExerciseAPIs.java | 546 ++++++++++++++++++ 1 file changed, 546 insertions(+) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingExerciseAPIs.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingExerciseAPIs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingExerciseAPIs.java new file mode 100644 index 0000000000..4335527d65 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingExerciseAPIs.java @@ -0,0 +1,546 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs; + +import com.google.common.collect.Lists; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.key.JavaKeyStoreProvider; +import org.apache.hadoop.fs.*; +import org.apache.hadoop.fs.permission.AclEntry; +import org.apache.hadoop.fs.permission.AclStatus; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.client.CreateEncryptionZoneFlag; +import org.apache.hadoop.hdfs.client.HdfsAdmin; +import org.apache.hadoop.hdfs.protocol.*; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite; +import org.apache.hadoop.io.erasurecode.ECSchema; +import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.file.Paths; +import java.security.NoSuchAlgorithmException; +import java.security.PrivilegedExceptionAction; +import java.util.EnumSet; +import java.util.List; +import java.util.Map; + +import static org.apache.hadoop.fs.permission.AclEntryScope.ACCESS; +import static org.apache.hadoop.fs.permission.AclEntryScope.DEFAULT; +import static org.apache.hadoop.fs.permission.AclEntryType.GROUP; +import static org.apache.hadoop.fs.permission.AclEntryType.OTHER; +import static org.apache.hadoop.fs.permission.AclEntryType.USER; +import static org.apache.hadoop.fs.permission.FsAction.ALL; +import static org.apache.hadoop.fs.permission.FsAction.NONE; +import static org.apache.hadoop.fs.permission.FsAction.READ_EXECUTE; +import static org.apache.hadoop.hdfs.server.namenode.AclTestHelpers.aclEntry; +import static org.junit.Assert.*; + +/** + * Test after enable Erasure Coding on cluster, exercise Java API make sure they + * are working as expected. + * + */ +public class TestErasureCodingExerciseAPIs { + private Configuration conf; + private MiniDFSCluster cluster; + private DistributedFileSystem fs; + private HdfsAdmin dfsAdmin; + private FileSystemTestWrapper fsWrapper; + private static final int BLOCK_SIZE = 1 << 14; // 16k + private ErasureCodingPolicy ecPolicy; + + private static ErasureCodingPolicy getEcPolicy() { + return StripedFileTestUtil.getDefaultECPolicy(); + } + + private static final Logger LOG = + LoggerFactory.getLogger(TestErasureCodingExerciseAPIs.class); + + + @Before + public void setupCluster() throws IOException { + ecPolicy = getEcPolicy(); + conf = new HdfsConfiguration(); + conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true); + + // Set up java key store + String testRootDir = Paths.get(new FileSystemTestHelper().getTestRootDir()) + .toString(); + String keyProviderURI = JavaKeyStoreProvider.SCHEME_NAME + "://file" + + new Path(testRootDir, "test.jks").toUri(); + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, + keyProviderURI); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, + true); + cluster = new MiniDFSCluster.Builder(conf). + numDataNodes(ecPolicy.getNumDataUnits() + ecPolicy.getNumParityUnits()). + build(); + cluster.waitActive(); + fs = cluster.getFileSystem(); + fsWrapper = new FileSystemTestWrapper(fs); + dfsAdmin = new HdfsAdmin(cluster.getURI(), conf); + DFSTestUtil.enableAllECPolicies(fs); + fs.setErasureCodingPolicy(new Path("/"), ecPolicy.getName()); + } + + /** + * FileSystem.[access, setOwner, setTime] API call should succeed without + * failure. + * @throws IOException if any IO operation failed. + * @throws InterruptedException + */ + @Test + public void testAccess() throws IOException, InterruptedException { + final Path p1 = new Path("/p1"); + final String userName = "user1"; + final String groupName = "group1"; + + fs.mkdir(p1, new FsPermission((short) 0444)); + fs.setOwner(p1, userName, groupName); + UserGroupInformation userGroupInfo = UserGroupInformation + .createUserForTesting(userName, new String[]{groupName}); + + FileSystem userFs = userGroupInfo.doAs( + (PrivilegedExceptionAction) () -> FileSystem.get(conf)); + + userFs.setOwner(p1, userName, groupName); + userFs.access(p1, FsAction.READ); + + long mtime = System.currentTimeMillis() - 1000L; + long atime = System.currentTimeMillis() - 2000L; + fs.setTimes(p1, mtime, atime); + FileStatus fileStatus = fs.getFileStatus(p1); + + assertEquals(userName, fileStatus.getOwner()); + assertEquals(groupName, fileStatus.getGroup()); + assertEquals(new FsPermission(FsAction.READ, FsAction.READ, FsAction.READ), + fileStatus.getPermission()); + assertEquals(mtime, fileStatus.getModificationTime()); + assertEquals(atime, fileStatus.getAccessTime()); + } + + /** + * FileSystem.[setQuota, getQuotaUsage, getContentSummary, + * setQuotaByStorageType] API call should succeed without failure. + * @throws IOException if any IO operation failed. + */ + @Test + public void testQuota() throws IOException { + final Path qDir = new Path("/quotaDir"); + fs.mkdirs(qDir); + fs.setQuota(qDir, 6, HdfsConstants.QUOTA_DONT_SET); + QuotaUsage usage = fs.getQuotaUsage(qDir); + assertEquals(fs.getContentSummary(qDir), usage); + + fs.setQuotaByStorageType(qDir, StorageType.DEFAULT, 10); + + } + + /** + * FileSystem.[addCachePool, modifyCachePool,removeCachePool] API call + * should without failure. FileSystem.[addCacheDirective, + * modifyCacheDirective, removeCacheDirective] are noop. + * @throws IOException if any IO operation failed. + */ + @Test + public void testCache() throws IOException { + fs.addCachePool(new CachePoolInfo("pool1")); + + fs.modifyCachePool(new CachePoolInfo("pool1")); + fs.removeCachePool("pool1"); + + fs.addCachePool(new CachePoolInfo("pool1")); + + // Below calls should be noop. + long id = fs.addCacheDirective(new CacheDirectiveInfo.Builder() + .setPool("pool1").setPath(new Path("/pool2")) + .build()); + RemoteIterator iter = fs.listCacheDirectives( + new CacheDirectiveInfo.Builder().setPool("pool1").build()); + assertTrue(iter.hasNext()); + assertEquals("pool1", iter.next().getInfo().getPool()); + + fs.modifyCacheDirective(new CacheDirectiveInfo.Builder() + .setId(id).setReplication((short) 2).build()); + fs.removeCacheDirective(id); + } + + /** + * FileSystem.[addErasureCodingPolicies, disableErasureCodingPolicy, + * getErasureCodingPolicy, removeErasureCodingPolicy, setErasureCodingPolicy + * unsetErasureCodingPolicy] API call still should be succeed without + * failure. + * @throws IOException if any IO operation failed. + */ + @Test + public void testErasureCodingPolicy() throws IOException { + final Path tDir = new Path("/ecpDir"); + fs.mkdirs(tDir); + ErasureCodingPolicy defaultPolicy + = SystemErasureCodingPolicies.getPolicies().get(0); + fs.setErasureCodingPolicy(tDir, defaultPolicy.getName()); + ErasureCodingPolicy fPolicy = fs.getErasureCodingPolicy(tDir); + assertEquals(defaultPolicy, fPolicy); + + final int cellSize = 1024 * 1024; + final ECSchema schema = new ECSchema("rs", 5, 3); + ErasureCodingPolicy newPolicy = + new ErasureCodingPolicy(schema, cellSize); + fs.addErasureCodingPolicies(new ErasureCodingPolicy[]{newPolicy}); + assertEquals(SystemErasureCodingPolicies.getPolicies().size() + 1, + fs.getAllErasureCodingPolicies().size()); + + fs.disableErasureCodingPolicy( + ErasureCodingPolicy.composePolicyName(schema, cellSize)); + assertEquals(SystemErasureCodingPolicies.getPolicies().size() + 1, + fs.getAllErasureCodingPolicies().size()); + + fs.unsetErasureCodingPolicy(tDir); + fPolicy = fs.getErasureCodingPolicy(tDir); + assertNotNull(fPolicy); + + fs.removeErasureCodingPolicy( + ErasureCodingPolicy.composePolicyName(schema, cellSize)); + assertEquals(SystemErasureCodingPolicies.getPolicies().size() + 1, + fs.getAllErasureCodingPolicies().size()); + } + + /** + * FileSystem.[getAclStatus, modifyAclEntries, removeAclEntries, removeAcl + * removeDefaultAcl] API call should succeed without failure. + * @throws IOException if any IO operation failed. + */ + @Test + public void testACLAPI() throws IOException { + Path p = new Path("/aclTest"); + fs.mkdirs(p, FsPermission.createImmutable((short) 0750)); + List aclSpec = Lists.newArrayList( + aclEntry(DEFAULT, USER, ALL), + aclEntry(DEFAULT, USER, "foo", ALL), + aclEntry(DEFAULT, GROUP, READ_EXECUTE), + aclEntry(DEFAULT, OTHER, NONE)); + fs.setAcl(p, aclSpec); + + AclStatus as = fs.getAclStatus(p); + + for (AclEntry entry : aclSpec) { + assertTrue(String.format("as: %s, entry: %s", as, entry), + as.getEntries().contains(entry)); + } + List maclSpec = Lists.newArrayList( + aclEntry(ACCESS, USER, "bar", READ_EXECUTE), + aclEntry(DEFAULT, USER, "bar", READ_EXECUTE)); + fs.modifyAclEntries(p, maclSpec); + + as = fs.getAclStatus(p); + for (AclEntry entry : maclSpec) { + assertTrue(String.format("as: %s, entry: %s", as, entry), + as.getEntries().contains(entry)); + } + + fs.removeAclEntries(p, maclSpec); + fs.removeDefaultAcl(p); + fs.removeAcl(p); + assertEquals(0, fs.getAclStatus(p).getEntries().size()); + } + + + /** + * FileSystem.[setXAttr, getXAttr, getXAttrs, removeXAttr, listXAttrs] API + * call should succeed without failure. + * @throws IOException if any IO operation failed. + */ + @Test + public void testAttr() throws IOException { + final Path p = new Path("/attrTest"); + fs.mkdirs(p); + final Path filePath = new Path(p, "file"); + try (DataOutputStream dos = fs.create(filePath)) { + dos.writeBytes("write something"); + } + + final String name = "user.a1"; + final byte[] value = {0x31, 0x32, 0x33}; + fs.setXAttr(filePath, name, value, EnumSet.of(XAttrSetFlag.CREATE)); + + Map xattrs = fs.getXAttrs(filePath); + assertEquals(1, xattrs.size()); + assertArrayEquals(value, xattrs.get(name)); + assertArrayEquals(value, fs.getXAttr(filePath, name)); + + List listXAttrs = fs.listXAttrs(filePath); + assertEquals(1, listXAttrs.size()); + + fs.removeXAttr(filePath, name); + + xattrs = fs.getXAttrs(filePath); + assertEquals(0, xattrs.size()); + listXAttrs = fs.listXAttrs(filePath); + assertEquals(0, listXAttrs.size()); + } + + /** + * FileSystem.[allowSnapshot, createSnapshot, deleteSnapshot, + * renameSnapshot, getSnapshotDiffReport, disallowSnapshot] API call should + * succeed without failure. + * @throws IOException if any IO operation failed. + */ + @Test + public void testSnapshotAPI() throws IOException { + Path p = new Path("/snapshotTest"); + fs.mkdirs(p); + fs.allowSnapshot(p); + + fs.createSnapshot(p, "s1"); + Path f = new Path("/snapshotTest/f1"); + try (DataOutputStream dos = fs.create(f)) { + dos.writeBytes("write something"); + } + + fs.createSnapshot(p, "s2"); + fs.renameSnapshot(p, "s2", "s3"); + SnapshotDiffReport report = fs.getSnapshotDiffReport(p, "s1", + "s3"); + assertEquals("s1", report.getFromSnapshot()); + assertEquals("s3", report.getLaterSnapshotName()); + + fs.deleteSnapshot(p, "s1"); + fs.deleteSnapshot(p, "s3"); + + fs.disallowSnapshot(p); + } + + /** + * FileSystem.[createSymlink, getFileLinkStatus] API call should succeed + * without failure. + * @throws IOException if any IO operation failed. + */ + @Test + public void testSymbolicLink() throws IOException { + Path p = new Path("/slTest"); + fs.mkdirs(p); + Path f = new Path("/slTest/file"); + try (DataOutputStream dos = fs.create(f)) { + dos.writeBytes("write something"); + } + + Path sl = new Path("/slTest1/sl"); + + fs.createSymlink(f, sl, true); + assertEquals(fs.getLinkTarget(sl), f); + FileStatus linkStatus = fs.getFileLinkStatus(sl); + assertTrue(linkStatus.isSymlink()); + } + + /** + * FileSystem.[create, open, append, concat, getFileChecksum, rename, + * delete] API call should succeed without failure. + * @throws IOException if any IO operation failed. + */ + @Test + public void testFileOpsAPI() throws IOException { + Path p = new Path("/fileTest"); + fs.mkdirs(p); + Path f1 = new Path(p, "file1"); + Path fa = new Path(p, "filea"); + + try (DataOutputStream dos = fs.create(f1)) { + dos.writeBytes("create with some content"); + } + + try (DataOutputStream dos = fs.create(fa)) { + dos.writeBytes("create with some content"); + } + + // setReplication is a noop + short replication = fs.getDefaultReplication(); + fs.setReplication(f1, (short) 5); + assertEquals(replication, fs.getDefaultReplication(f1)); + + BlockLocation[] locations = fs.getFileBlockLocations(f1, 0, 1); + assertEquals(1, locations.length); + + FileStatus status1 = fs.getFileStatus(f1); + assertFalse(status1.isDirectory()); + assertTrue(status1.getPath().toString().contains(p.toString())); + FileStatus statusa = fs.getFileStatus(fa); + assertFalse(statusa.isDirectory()); + assertTrue(statusa.getPath().toString().contains(fa.toString())); + + FileStatus[] statuses = fs.listStatus(p); + assertEquals(2, statuses.length); + assertEquals(status1, statuses[0]); + assertEquals(statusa, statuses[1]); + + RemoteIterator iter = fs.listStatusIterator(p); + assertEquals(status1, iter.next()); + assertEquals(statusa, iter.next()); + assertFalse(iter.hasNext()); + + Path[] concatPs = new Path[]{ + new Path(p, "c1"), + new Path(p, "c2"), + new Path(p, "c3"), + }; + + for (Path cp : concatPs) { + try (DataOutputStream dos = fs.create(cp)) { + dos.writeBytes("concat some content"); + } + } + fs.concat(f1, concatPs); + + FileChecksum checksum1 = fs.getFileChecksum(f1); + Path f2 = new Path("/fileTest/file2"); + + fs.rename(f1, f2); + FileStatus fileStatus = fs.getFileStatus(f2); + assertTrue(fileStatus.getPath().toString().contains("/fileTest/file2")); + + FileChecksum checksum2 = fs.getFileChecksum(f2); + assertEquals(checksum1, checksum2); + fs.delete(f2, true); + + RemoteIterator corruptFileBlocks = fs.listCorruptFileBlocks(f2); + assertFalse(corruptFileBlocks.hasNext()); + } + + + /** + * FileSystem.[createEncryptionZone, getLocatedBlocks, getEZForPath, + * reencryptEncryptionZone, addDelegationTokens] API call should succeed + * without failure. + * @throws IOException if any IO operation failed. + * @throws NoSuchAlgorithmException + */ + @Test + public void testEncryptionZone() throws IOException, + NoSuchAlgorithmException { + final Path zoneRoot = new Path("ecRoot"); + final Path zonePath = new Path(zoneRoot, "/ec"); + fsWrapper.mkdir(zonePath, FsPermission.getDirDefault(), true); + + final String testKey = "test_key"; + DFSTestUtil.createKey(testKey, cluster, conf); + final EnumSet noTrash = + EnumSet.of(CreateEncryptionZoneFlag.NO_TRASH); + dfsAdmin.createEncryptionZone(zonePath, testKey, noTrash); + + final Path fp = new Path(zonePath, "encFile"); + DFSTestUtil.createFile(fs, fp, 1 << 13, (short) 1, 0xFEEE); + LocatedBlocks blocks = fs.getClient().getLocatedBlocks(fp.toString(), 0); + FileEncryptionInfo fei = blocks.getFileEncryptionInfo(); + assertEquals(testKey, fei.getKeyName()); + EncryptionZone ez = fs.getEZForPath(fp); + + assertEquals(zonePath.toString(), ez.getPath()); + dfsAdmin.reencryptEncryptionZone(zonePath, + HdfsConstants.ReencryptAction.START); + + Credentials creds = new Credentials(); + final Token[] tokens = fs.addDelegationTokens("JobTracker", creds); + assertEquals(1, tokens.length); + } + + /** + * FileSystem.[setStoragePolicy, unsetStoragePolicy] API call should succeed + * without failure. + * @throws IOException if any IO operation failed. + */ + @Test + public void testStoragePolicy() throws IOException { + Path p = new Path("/storagePolicyTest"); + fs.mkdirs(p); + final Path sp = new Path(p, "/sp"); + try (DataOutputStream dos = fs.create(sp)) { + dos.writeBytes("create with some content"); + } + + final BlockStoragePolicySuite suite = BlockStoragePolicySuite + .createDefaultSuite(); + final BlockStoragePolicy hot = suite.getPolicy("HOT"); + + fs.setStoragePolicy(sp, hot.getName()); + assertEquals(fs.getStoragePolicy(sp), hot); + + fs.unsetStoragePolicy(sp); + assertEquals(fs.getStoragePolicy(sp), hot); + } + + /** + * append is not supported in EC. + * @throws IOException if any IO operation failed. + */ + @Test + public void testAppend() throws IOException { + Path p = new Path("/fileTest"); + fs.mkdirs(p); + Path f = new Path("/fileTest/appendFile"); + + try (DataOutputStream dos = fs.create(f)) { + dos.writeBytes("create with some content"); + } + + try { + fs.append(f); + fail("append is not supported on erasure coded file"); + } catch (IOException ioe) { + //Work as expected. + } + } + + + /** + * truncate is not supported in EC. + * @throws IOException if any IO operation failed. + */ + @Test + public void testTruncate() throws IOException { + Path p = new Path("/truncateTest"); + fs.mkdirs(p); + Path f = new Path("/truncateTest/truncatefile"); + try (DataOutputStream dos = fs.create(f)) { + dos.writeBytes("create with some content"); + } + + try { + fs.truncate(f, 0); + fail("truncate is not supported on erasure coded file."); + } catch (IOException ex) { + //Work as expected. + } + } + + @After + public void shutdownCluster() { + if (cluster != null) { + cluster.shutdown(); + cluster = null; + } + } +}