big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-13109. Support fully qualified hdfs path in EZ commands. Contributed by Hanisha Koneru.

Browse Source
This commit is contained in:
Xiaoyu Yao 2018-03-06 16:44:20 -08:00
parent 346caa2095
commit edf9445708
3 changed files with 111 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
View File

@ -37,6 +37,7 @@
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FSDataOutputStreamBuilder;
import org.apache.hadoop.fs.FSLinkResolver;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.fs.FileStatus;
@ -2600,6 +2601,70 @@ public FileEncryptionInfo next(final FileSystem fs, final Path p)
}.resolve(this, absF);
}
/* HDFS only */
public void provisionEZTrash(final Path path,
final FsPermission trashPermission) throws IOException {
Path absF = fixRelativePart(path);
new FileSystemLinkResolver<Void>() {
@Override
public Void doCall(Path p) throws IOException {
provisionEZTrash(getPathName(p), trashPermission);
return null;
}
@Override
public Void next(FileSystem fs, Path p) throws IOException {
if (fs instanceof DistributedFileSystem) {
DistributedFileSystem myDfs = (DistributedFileSystem)fs;
myDfs.provisionEZTrash(p, trashPermission);
return null;
}
throw new UnsupportedOperationException("Cannot provisionEZTrash " +
"through a symlink to a non-DistributedFileSystem: " + fs + " -> "
+ p);
}
}.resolve(this, absF);
}
private void provisionEZTrash(String path, FsPermission trashPermission)
throws IOException {
// make sure the path is an EZ
EncryptionZone ez = dfs.getEZForPath(path);
if (ez == null) {
throw new IllegalArgumentException(path + " is not an encryption zone.");
}
String ezPath = ez.getPath();
if (!path.toString().equals(ezPath)) {
throw new IllegalArgumentException(path + " is not the root of an " +
"encryption zone. Do you mean " + ez.getPath() + "?");
}
// check if the trash directory exists
Path trashPath = new Path(ez.getPath(), FileSystem.TRASH_PREFIX);
try {
FileStatus trashFileStatus = getFileStatus(trashPath);
String errMessage = "Will not provision new trash directory for " +
"encryption zone " + ez.getPath() + ". Path already exists.";
if (!trashFileStatus.isDirectory()) {
errMessage += "\r\n" +
"Warning: " + trashPath.toString() + " is not a directory";
}
if (!trashFileStatus.getPermission().equals(trashPermission)) {
errMessage += "\r\n" +
"Warning: the permission of " +
trashPath.toString() + " is not " + trashPermission;
}
throw new FileAlreadyExistsException(errMessage);
} catch (FileNotFoundException ignored) {
// no trash path
}
// Update the permission bits
mkdir(trashPath, trashPermission);
setPermission(trashPath, trashPermission);
}
@Override
public void setXAttr(Path path, final String name, final byte[] value,
final EnumSet<XAttrSetFlag> flag) throws IOException {

46
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
View File

@ -24,9 +24,7 @@
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.fs.BlockStoragePolicySpi;
import org.apache.hadoop.fs.CacheFlag;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
@ -330,7 +328,7 @@ public void createEncryptionZone(Path path, String keyName,
throw new HadoopIllegalArgumentException(
"can not have both PROVISION_TRASH and NO_TRASH flags");
}
this.provisionEZTrash(path);
dfs.provisionEZTrash(path, TRASH_PERMISSION);
}
}
@ -341,7 +339,7 @@ public void createEncryptionZone(Path path, String keyName,
* @throws IOException if the trash directory can not be created.
*/
public void provisionEncryptionZoneTrash(Path path) throws IOException {
this.provisionEZTrash(path);
dfs.provisionEZTrash(path, TRASH_PERMISSION);
}
/**
@ -603,46 +601,6 @@ public void disableErasureCodingPolicy(String ecPolicyName)
dfs.disableErasureCodingPolicy(ecPolicyName);
}
private void provisionEZTrash(Path path) throws IOException {
// make sure the path is an EZ
EncryptionZone ez = dfs.getEZForPath(path);
if (ez == null) {
throw new IllegalArgumentException(path + " is not an encryption zone.");
}
String ezPath = ez.getPath();
if (!path.toString().equals(ezPath)) {
throw new IllegalArgumentException(path + " is not the root of an " +
"encryption zone. Do you mean " + ez.getPath() + "?");
}
// check if the trash directory exists
Path trashPath = new Path(ez.getPath(), FileSystem.TRASH_PREFIX);
try {
FileStatus trashFileStatus = dfs.getFileStatus(trashPath);
String errMessage = "Will not provision new trash directory for " +
"encryption zone " + ez.getPath() + ". Path already exists.";
if (!trashFileStatus.isDirectory()) {
errMessage += "\r\n" +
"Warning: " + trashPath.toString() + " is not a directory";
}
if (!trashFileStatus.getPermission().equals(TRASH_PERMISSION)) {
errMessage += "\r\n" +
"Warning: the permission of " +
trashPath.toString() + " is not " + TRASH_PERMISSION;
}
throw new FileAlreadyExistsException(errMessage);
} catch (FileNotFoundException ignored) {
// no trash path
}
// Update the permission bits
dfs.mkdir(trashPath, TRASH_PERMISSION);
dfs.setPermission(trashPath, TRASH_PERMISSION);
}
/**
* Returns a RemoteIterator which can be used to list all open files
* currently managed by the NameNode. For large numbers of open files,

44
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
View File

@ -540,6 +540,50 @@ public void testBasicOperationsRootDir() throws Exception {
assertZonePresent(null, rootDir.toString());
}
@Test
public void testEZwithFullyQualifiedPath() throws Exception {
/* Test failure of create EZ on a directory that doesn't exist. */
final Path zoneParent = new Path("/zones");
final Path zone1 = new Path(zoneParent, "zone1");
final Path zone1FQP = new Path(cluster.getURI().toString(), zone1);
final Path zone2 = new Path(zoneParent, "zone2");
final Path zone2FQP = new Path(cluster.getURI().toString(), zone2);
int numZones = 0;
EnumSet<CreateEncryptionZoneFlag> withTrash = EnumSet
.of(CreateEncryptionZoneFlag.PROVISION_TRASH);
// Create EZ with Trash using FQP
fsWrapper.mkdir(zone1FQP, FsPermission.getDirDefault(), true);
dfsAdmin.createEncryptionZone(zone1FQP, TEST_KEY, withTrash);
assertNumZones(++numZones);
assertZonePresent(TEST_KEY, zone1.toString());
// Check that zone1 contains a .Trash directory
final Path zone1Trash = new Path(zone1, fs.TRASH_PREFIX);
assertTrue("CreateEncryptionZone with trash enabled should create a " +
".Trash directory in the EZ", fs.exists(zone1Trash));
// getEncryptionZoneForPath for FQP should return the path component
EncryptionZone ezForZone1 = dfsAdmin.getEncryptionZoneForPath(zone1FQP);
assertTrue("getEncryptionZoneForPath for fully qualified path should " +
"return the path component",
ezForZone1.getPath().equals(zone1.toString()));
// Create EZ without Trash
fsWrapper.mkdir(zone2FQP, FsPermission.getDirDefault(), true);
dfsAdmin.createEncryptionZone(zone2FQP, TEST_KEY, NO_TRASH);
assertNumZones(++numZones);
assertZonePresent(TEST_KEY, zone2.toString());
// Provision Trash on zone2 using FQP
dfsAdmin.provisionEncryptionZoneTrash(zone2FQP);
EncryptionZone ezForZone2 = dfsAdmin.getEncryptionZoneForPath(zone2FQP);
Path ezTrashForZone2 = new Path(ezForZone2.getPath(),
FileSystem.TRASH_PREFIX);
assertTrue("provisionEZTrash with fully qualified path should create " +
"trash directory ", fsWrapper.exists(ezTrashForZone2));
}
/**
* Test listing encryption zones as a non super user.
*/