From edf9445708ffb7a9e59cb933e049b540f99add1e Mon Sep 17 00:00:00 2001 From: Xiaoyu Yao Date: Tue, 6 Mar 2018 16:44:20 -0800 Subject: [PATCH] HDFS-13109. Support fully qualified hdfs path in EZ commands. Contributed by Hanisha Koneru. --- .../hadoop/hdfs/DistributedFileSystem.java | 65 +++++++++++++++++++ .../apache/hadoop/hdfs/client/HdfsAdmin.java | 46 +------------ .../hadoop/hdfs/TestEncryptionZones.java | 44 +++++++++++++ 3 files changed, 111 insertions(+), 44 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index fe2a5653bb..03cb317b58 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -37,6 +37,7 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FSDataOutputStreamBuilder; import org.apache.hadoop.fs.FSLinkResolver; +import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.fs.FileStatus; @@ -2600,6 +2601,70 @@ public FileEncryptionInfo next(final FileSystem fs, final Path p) }.resolve(this, absF); } + /* HDFS only */ + public void provisionEZTrash(final Path path, + final FsPermission trashPermission) throws IOException { + Path absF = fixRelativePart(path); + new FileSystemLinkResolver() { + @Override + public Void doCall(Path p) throws IOException { + provisionEZTrash(getPathName(p), trashPermission); + return null; + } + + @Override + public Void next(FileSystem fs, Path p) throws IOException { + if (fs instanceof DistributedFileSystem) { + DistributedFileSystem myDfs = (DistributedFileSystem)fs; + myDfs.provisionEZTrash(p, trashPermission); + return null; + } + throw new UnsupportedOperationException("Cannot provisionEZTrash " + + "through a symlink to a non-DistributedFileSystem: " + fs + " -> " + + p); + } + }.resolve(this, absF); + } + + private void provisionEZTrash(String path, FsPermission trashPermission) + throws IOException { + // make sure the path is an EZ + EncryptionZone ez = dfs.getEZForPath(path); + if (ez == null) { + throw new IllegalArgumentException(path + " is not an encryption zone."); + } + + String ezPath = ez.getPath(); + if (!path.toString().equals(ezPath)) { + throw new IllegalArgumentException(path + " is not the root of an " + + "encryption zone. Do you mean " + ez.getPath() + "?"); + } + + // check if the trash directory exists + Path trashPath = new Path(ez.getPath(), FileSystem.TRASH_PREFIX); + try { + FileStatus trashFileStatus = getFileStatus(trashPath); + String errMessage = "Will not provision new trash directory for " + + "encryption zone " + ez.getPath() + ". Path already exists."; + if (!trashFileStatus.isDirectory()) { + errMessage += "\r\n" + + "Warning: " + trashPath.toString() + " is not a directory"; + } + if (!trashFileStatus.getPermission().equals(trashPermission)) { + errMessage += "\r\n" + + "Warning: the permission of " + + trashPath.toString() + " is not " + trashPermission; + } + throw new FileAlreadyExistsException(errMessage); + } catch (FileNotFoundException ignored) { + // no trash path + } + + // Update the permission bits + mkdir(trashPath, trashPermission); + setPermission(trashPath, trashPermission); + } + @Override public void setXAttr(Path path, final String name, final byte[] value, final EnumSet flag) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java index 2c0659a206..88044b9714 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java @@ -24,9 +24,7 @@ import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.fs.BlockStoragePolicySpi; import org.apache.hadoop.fs.CacheFlag; -import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileEncryptionInfo; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; @@ -330,7 +328,7 @@ public void createEncryptionZone(Path path, String keyName, throw new HadoopIllegalArgumentException( "can not have both PROVISION_TRASH and NO_TRASH flags"); } - this.provisionEZTrash(path); + dfs.provisionEZTrash(path, TRASH_PERMISSION); } } @@ -341,7 +339,7 @@ public void createEncryptionZone(Path path, String keyName, * @throws IOException if the trash directory can not be created. */ public void provisionEncryptionZoneTrash(Path path) throws IOException { - this.provisionEZTrash(path); + dfs.provisionEZTrash(path, TRASH_PERMISSION); } /** @@ -603,46 +601,6 @@ public void disableErasureCodingPolicy(String ecPolicyName) dfs.disableErasureCodingPolicy(ecPolicyName); } - private void provisionEZTrash(Path path) throws IOException { - // make sure the path is an EZ - EncryptionZone ez = dfs.getEZForPath(path); - if (ez == null) { - throw new IllegalArgumentException(path + " is not an encryption zone."); - } - - String ezPath = ez.getPath(); - if (!path.toString().equals(ezPath)) { - throw new IllegalArgumentException(path + " is not the root of an " + - "encryption zone. Do you mean " + ez.getPath() + "?"); - } - - // check if the trash directory exists - - Path trashPath = new Path(ez.getPath(), FileSystem.TRASH_PREFIX); - - try { - FileStatus trashFileStatus = dfs.getFileStatus(trashPath); - String errMessage = "Will not provision new trash directory for " + - "encryption zone " + ez.getPath() + ". Path already exists."; - if (!trashFileStatus.isDirectory()) { - errMessage += "\r\n" + - "Warning: " + trashPath.toString() + " is not a directory"; - } - if (!trashFileStatus.getPermission().equals(TRASH_PERMISSION)) { - errMessage += "\r\n" + - "Warning: the permission of " + - trashPath.toString() + " is not " + TRASH_PERMISSION; - } - throw new FileAlreadyExistsException(errMessage); - } catch (FileNotFoundException ignored) { - // no trash path - } - - // Update the permission bits - dfs.mkdir(trashPath, TRASH_PERMISSION); - dfs.setPermission(trashPath, TRASH_PERMISSION); - } - /** * Returns a RemoteIterator which can be used to list all open files * currently managed by the NameNode. For large numbers of open files, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java index e2d307a4e1..4e69b9d99a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java @@ -540,6 +540,50 @@ public void testBasicOperationsRootDir() throws Exception { assertZonePresent(null, rootDir.toString()); } + @Test + public void testEZwithFullyQualifiedPath() throws Exception { + /* Test failure of create EZ on a directory that doesn't exist. */ + final Path zoneParent = new Path("/zones"); + final Path zone1 = new Path(zoneParent, "zone1"); + final Path zone1FQP = new Path(cluster.getURI().toString(), zone1); + final Path zone2 = new Path(zoneParent, "zone2"); + final Path zone2FQP = new Path(cluster.getURI().toString(), zone2); + + int numZones = 0; + EnumSet withTrash = EnumSet + .of(CreateEncryptionZoneFlag.PROVISION_TRASH); + + // Create EZ with Trash using FQP + fsWrapper.mkdir(zone1FQP, FsPermission.getDirDefault(), true); + dfsAdmin.createEncryptionZone(zone1FQP, TEST_KEY, withTrash); + assertNumZones(++numZones); + assertZonePresent(TEST_KEY, zone1.toString()); + // Check that zone1 contains a .Trash directory + final Path zone1Trash = new Path(zone1, fs.TRASH_PREFIX); + assertTrue("CreateEncryptionZone with trash enabled should create a " + + ".Trash directory in the EZ", fs.exists(zone1Trash)); + + // getEncryptionZoneForPath for FQP should return the path component + EncryptionZone ezForZone1 = dfsAdmin.getEncryptionZoneForPath(zone1FQP); + assertTrue("getEncryptionZoneForPath for fully qualified path should " + + "return the path component", + ezForZone1.getPath().equals(zone1.toString())); + + // Create EZ without Trash + fsWrapper.mkdir(zone2FQP, FsPermission.getDirDefault(), true); + dfsAdmin.createEncryptionZone(zone2FQP, TEST_KEY, NO_TRASH); + assertNumZones(++numZones); + assertZonePresent(TEST_KEY, zone2.toString()); + + // Provision Trash on zone2 using FQP + dfsAdmin.provisionEncryptionZoneTrash(zone2FQP); + EncryptionZone ezForZone2 = dfsAdmin.getEncryptionZoneForPath(zone2FQP); + Path ezTrashForZone2 = new Path(ezForZone2.getPath(), + FileSystem.TRASH_PREFIX); + assertTrue("provisionEZTrash with fully qualified path should create " + + "trash directory ", fsWrapper.exists(ezTrashForZone2)); + } + /** * Test listing encryption zones as a non super user. */