HDFS-8112. Relax permission checking for EC related operations.
This commit is contained in:
parent
490abfb10f
commit
3085a60430
@ -40,6 +40,7 @@
|
|||||||
import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp;
|
import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp;
|
||||||
import org.apache.hadoop.io.IOUtils;
|
import org.apache.hadoop.io.IOUtils;
|
||||||
import org.apache.hadoop.io.WritableUtils;
|
import org.apache.hadoop.io.WritableUtils;
|
||||||
|
import org.apache.hadoop.security.AccessControlException;
|
||||||
|
|
||||||
import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.XATTR_ERASURECODING_POLICY;
|
import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.XATTR_ERASURECODING_POLICY;
|
||||||
|
|
||||||
@ -66,15 +67,15 @@ private FSDirErasureCodingOp() {}
|
|||||||
* @return {@link HdfsFileStatus}
|
* @return {@link HdfsFileStatus}
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
* @throws HadoopIllegalArgumentException if the policy is not enabled
|
* @throws HadoopIllegalArgumentException if the policy is not enabled
|
||||||
|
* @throws AccessControlException if the user does not have write access
|
||||||
*/
|
*/
|
||||||
static HdfsFileStatus setErasureCodingPolicy(final FSNamesystem fsn,
|
static HdfsFileStatus setErasureCodingPolicy(final FSNamesystem fsn,
|
||||||
final String srcArg, final String ecPolicyName,
|
final String srcArg, final String ecPolicyName,
|
||||||
final boolean logRetryCache) throws IOException {
|
final FSPermissionChecker pc, final boolean logRetryCache)
|
||||||
|
throws IOException, AccessControlException {
|
||||||
assert fsn.hasWriteLock();
|
assert fsn.hasWriteLock();
|
||||||
|
|
||||||
String src = srcArg;
|
String src = srcArg;
|
||||||
FSPermissionChecker pc = null;
|
|
||||||
pc = fsn.getPermissionChecker();
|
|
||||||
FSDirectory fsd = fsn.getFSDirectory();
|
FSDirectory fsd = fsn.getFSDirectory();
|
||||||
final INodesInPath iip;
|
final INodesInPath iip;
|
||||||
List<XAttr> xAttrs;
|
List<XAttr> xAttrs;
|
||||||
@ -88,6 +89,10 @@ static HdfsFileStatus setErasureCodingPolicy(final FSNamesystem fsn,
|
|||||||
"policies.");
|
"policies.");
|
||||||
}
|
}
|
||||||
iip = fsd.resolvePath(pc, src, DirOp.WRITE_LINK);
|
iip = fsd.resolvePath(pc, src, DirOp.WRITE_LINK);
|
||||||
|
// Write access is required to set erasure coding policy
|
||||||
|
if (fsd.isPermissionEnabled()) {
|
||||||
|
fsd.checkPathAccess(pc, iip, FsAction.WRITE);
|
||||||
|
}
|
||||||
src = iip.getPath();
|
src = iip.getPath();
|
||||||
xAttrs = setErasureCodingPolicyXAttr(fsn, iip, ecPolicy);
|
xAttrs = setErasureCodingPolicyXAttr(fsn, iip, ecPolicy);
|
||||||
} finally {
|
} finally {
|
||||||
@ -97,7 +102,7 @@ static HdfsFileStatus setErasureCodingPolicy(final FSNamesystem fsn,
|
|||||||
return fsd.getAuditFileInfo(iip);
|
return fsd.getAuditFileInfo(iip);
|
||||||
}
|
}
|
||||||
|
|
||||||
static List<XAttr> setErasureCodingPolicyXAttr(final FSNamesystem fsn,
|
private static List<XAttr> setErasureCodingPolicyXAttr(final FSNamesystem fsn,
|
||||||
final INodesInPath srcIIP, ErasureCodingPolicy ecPolicy) throws IOException {
|
final INodesInPath srcIIP, ErasureCodingPolicy ecPolicy) throws IOException {
|
||||||
FSDirectory fsd = fsn.getFSDirectory();
|
FSDirectory fsd = fsn.getFSDirectory();
|
||||||
assert fsd.hasWriteLock();
|
assert fsd.hasWriteLock();
|
||||||
@ -165,19 +170,24 @@ static List<XAttr> setErasureCodingPolicyXAttr(final FSNamesystem fsn,
|
|||||||
* cache rebuilding
|
* cache rebuilding
|
||||||
* @return {@link HdfsFileStatus}
|
* @return {@link HdfsFileStatus}
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
|
* @throws AccessControlException if the user does not have write access
|
||||||
*/
|
*/
|
||||||
static HdfsFileStatus unsetErasureCodingPolicy(final FSNamesystem fsn,
|
static HdfsFileStatus unsetErasureCodingPolicy(final FSNamesystem fsn,
|
||||||
final String srcArg, final boolean logRetryCache) throws IOException {
|
final String srcArg, final FSPermissionChecker pc,
|
||||||
|
final boolean logRetryCache) throws IOException {
|
||||||
assert fsn.hasWriteLock();
|
assert fsn.hasWriteLock();
|
||||||
|
|
||||||
String src = srcArg;
|
String src = srcArg;
|
||||||
FSPermissionChecker pc = fsn.getPermissionChecker();
|
|
||||||
FSDirectory fsd = fsn.getFSDirectory();
|
FSDirectory fsd = fsn.getFSDirectory();
|
||||||
final INodesInPath iip;
|
final INodesInPath iip;
|
||||||
List<XAttr> xAttrs;
|
List<XAttr> xAttrs;
|
||||||
fsd.writeLock();
|
fsd.writeLock();
|
||||||
try {
|
try {
|
||||||
iip = fsd.resolvePath(pc, src, DirOp.WRITE_LINK);
|
iip = fsd.resolvePath(pc, src, DirOp.WRITE_LINK);
|
||||||
|
// Write access is required to unset erasure coding policy
|
||||||
|
if (fsd.isPermissionEnabled()) {
|
||||||
|
fsd.checkPathAccess(pc, iip, FsAction.WRITE);
|
||||||
|
}
|
||||||
src = iip.getPath();
|
src = iip.getPath();
|
||||||
xAttrs = removeErasureCodingPolicyXAttr(fsn, iip);
|
xAttrs = removeErasureCodingPolicyXAttr(fsn, iip);
|
||||||
} finally {
|
} finally {
|
||||||
@ -225,29 +235,23 @@ private static List<XAttr> removeErasureCodingPolicyXAttr(
|
|||||||
* @return {@link ErasureCodingPolicy}
|
* @return {@link ErasureCodingPolicy}
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
* @throws FileNotFoundException if the path does not exist.
|
* @throws FileNotFoundException if the path does not exist.
|
||||||
|
* @throws AccessControlException if no read access
|
||||||
*/
|
*/
|
||||||
static ErasureCodingPolicy getErasureCodingPolicy(final FSNamesystem fsn,
|
static ErasureCodingPolicy getErasureCodingPolicy(final FSNamesystem fsn,
|
||||||
final String src) throws IOException {
|
final String src, FSPermissionChecker pc)
|
||||||
|
throws IOException, AccessControlException {
|
||||||
assert fsn.hasReadLock();
|
assert fsn.hasReadLock();
|
||||||
|
|
||||||
final INodesInPath iip = getINodesInPath(fsn, src);
|
FSDirectory fsd = fsn.getFSDirectory();
|
||||||
|
final INodesInPath iip = fsd.resolvePath(pc, src, DirOp.READ);
|
||||||
|
if (fsn.isPermissionEnabled()) {
|
||||||
|
fsn.getFSDirectory().checkPathAccess(pc, iip, FsAction.READ);
|
||||||
|
}
|
||||||
|
|
||||||
if (iip.getLastINode() == null) {
|
if (iip.getLastINode() == null) {
|
||||||
throw new FileNotFoundException("Path not found: " + iip.getPath());
|
throw new FileNotFoundException("Path not found: " + iip.getPath());
|
||||||
}
|
}
|
||||||
return getErasureCodingPolicyForPath(fsn, iip);
|
return getErasureCodingPolicyForPath(fsd, iip);
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if the file or directory has an erasure coding policy.
|
|
||||||
*
|
|
||||||
* @param fsn namespace
|
|
||||||
* @param srcArg path
|
|
||||||
* @return Whether the file or directory has an erasure coding policy.
|
|
||||||
* @throws IOException
|
|
||||||
*/
|
|
||||||
static boolean hasErasureCodingPolicy(final FSNamesystem fsn,
|
|
||||||
final String srcArg) throws IOException {
|
|
||||||
return hasErasureCodingPolicy(fsn, getINodesInPath(fsn, srcArg));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -260,22 +264,22 @@ static boolean hasErasureCodingPolicy(final FSNamesystem fsn,
|
|||||||
*/
|
*/
|
||||||
static boolean hasErasureCodingPolicy(final FSNamesystem fsn,
|
static boolean hasErasureCodingPolicy(final FSNamesystem fsn,
|
||||||
final INodesInPath iip) throws IOException {
|
final INodesInPath iip) throws IOException {
|
||||||
return getErasureCodingPolicy(fsn, iip) != null;
|
return unprotectedGetErasureCodingPolicy(fsn, iip) != null;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get the erasure coding policy.
|
* Get the erasure coding policy. This does not do any permission checking.
|
||||||
*
|
*
|
||||||
* @param fsn namespace
|
* @param fsn namespace
|
||||||
* @param iip inodes in the path containing the file
|
* @param iip inodes in the path containing the file
|
||||||
* @return {@link ErasureCodingPolicy}
|
* @return {@link ErasureCodingPolicy}
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
static ErasureCodingPolicy getErasureCodingPolicy(final FSNamesystem fsn,
|
static ErasureCodingPolicy unprotectedGetErasureCodingPolicy(
|
||||||
final INodesInPath iip) throws IOException {
|
final FSNamesystem fsn, final INodesInPath iip) throws IOException {
|
||||||
assert fsn.hasReadLock();
|
assert fsn.hasReadLock();
|
||||||
|
|
||||||
return getErasureCodingPolicyForPath(fsn, iip);
|
return getErasureCodingPolicyForPath(fsn.getFSDirectory(), iip);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -290,21 +294,9 @@ static ErasureCodingPolicy[] getErasureCodingPolicies(final FSNamesystem fsn)
|
|||||||
return fsn.getErasureCodingPolicyManager().getPolicies();
|
return fsn.getErasureCodingPolicyManager().getPolicies();
|
||||||
}
|
}
|
||||||
|
|
||||||
private static INodesInPath getINodesInPath(final FSNamesystem fsn,
|
private static ErasureCodingPolicy getErasureCodingPolicyForPath(
|
||||||
final String srcArg) throws IOException {
|
FSDirectory fsd, INodesInPath iip) throws IOException {
|
||||||
final FSDirectory fsd = fsn.getFSDirectory();
|
|
||||||
final FSPermissionChecker pc = fsn.getPermissionChecker();
|
|
||||||
INodesInPath iip = fsd.resolvePath(pc, srcArg, DirOp.READ);
|
|
||||||
if (fsn.isPermissionEnabled()) {
|
|
||||||
fsn.getFSDirectory().checkPathAccess(pc, iip, FsAction.READ);
|
|
||||||
}
|
|
||||||
return iip;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static ErasureCodingPolicy getErasureCodingPolicyForPath(FSNamesystem fsn,
|
|
||||||
INodesInPath iip) throws IOException {
|
|
||||||
Preconditions.checkNotNull(iip, "INodes cannot be null");
|
Preconditions.checkNotNull(iip, "INodes cannot be null");
|
||||||
FSDirectory fsd = fsn.getFSDirectory();
|
|
||||||
fsd.readLock();
|
fsd.readLock();
|
||||||
try {
|
try {
|
||||||
List<INode> inodes = iip.getReadOnlyINodes();
|
List<INode> inodes = iip.getReadOnlyINodes();
|
||||||
|
@ -172,7 +172,7 @@ static GetBlockLocationsResult getBlockLocations(
|
|||||||
final FileEncryptionInfo feInfo =
|
final FileEncryptionInfo feInfo =
|
||||||
FSDirEncryptionZoneOp.getFileEncryptionInfo(fsd, iip);
|
FSDirEncryptionZoneOp.getFileEncryptionInfo(fsd, iip);
|
||||||
final ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.
|
final ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.
|
||||||
getErasureCodingPolicy(fsd.getFSNamesystem(), iip);
|
unprotectedGetErasureCodingPolicy(fsd.getFSNamesystem(), iip);
|
||||||
|
|
||||||
final LocatedBlocks blocks = bm.createLocatedBlocks(
|
final LocatedBlocks blocks = bm.createLocatedBlocks(
|
||||||
inode.getBlocks(iip.getPathSnapshotId()), fileSize, isUc, offset,
|
inode.getBlocks(iip.getPathSnapshotId()), fileSize, isUc, offset,
|
||||||
@ -413,7 +413,7 @@ private static HdfsFileStatus createFileStatus(
|
|||||||
FileEncryptionInfo feInfo = null;
|
FileEncryptionInfo feInfo = null;
|
||||||
|
|
||||||
final ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp
|
final ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp
|
||||||
.getErasureCodingPolicy(fsd.getFSNamesystem(), iip);
|
.unprotectedGetErasureCodingPolicy(fsd.getFSNamesystem(), iip);
|
||||||
|
|
||||||
if (node.isFile()) {
|
if (node.isFile()) {
|
||||||
final INodeFile fileNode = node.asFile();
|
final INodeFile fileNode = node.asFile();
|
||||||
|
@ -190,7 +190,8 @@ static ValidateAddBlockResult validateAddBlock(
|
|||||||
blockType = pendingFile.getBlockType();
|
blockType = pendingFile.getBlockType();
|
||||||
ErasureCodingPolicy ecPolicy = null;
|
ErasureCodingPolicy ecPolicy = null;
|
||||||
if (blockType == BlockType.STRIPED) {
|
if (blockType == BlockType.STRIPED) {
|
||||||
ecPolicy = FSDirErasureCodingOp.getErasureCodingPolicy(fsn, src);
|
ecPolicy =
|
||||||
|
FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy(fsn, iip);
|
||||||
numTargets = (short) (ecPolicy.getSchema().getNumDataUnits()
|
numTargets = (short) (ecPolicy.getSchema().getNumDataUnits()
|
||||||
+ ecPolicy.getSchema().getNumParityUnits());
|
+ ecPolicy.getSchema().getNumParityUnits());
|
||||||
} else {
|
} else {
|
||||||
@ -418,7 +419,7 @@ static INodeFile addFileForEditLog(
|
|||||||
// check if the file has an EC policy
|
// check if the file has an EC policy
|
||||||
boolean isStriped = false;
|
boolean isStriped = false;
|
||||||
ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.
|
ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.
|
||||||
getErasureCodingPolicy(fsd.getFSNamesystem(), existing);
|
unprotectedGetErasureCodingPolicy(fsd.getFSNamesystem(), existing);
|
||||||
if (ecPolicy != null) {
|
if (ecPolicy != null) {
|
||||||
isStriped = true;
|
isStriped = true;
|
||||||
}
|
}
|
||||||
@ -475,8 +476,9 @@ private static BlockInfo addBlock(FSDirectory fsd, String path,
|
|||||||
// associate new last block for the file
|
// associate new last block for the file
|
||||||
final BlockInfo blockInfo;
|
final BlockInfo blockInfo;
|
||||||
if (blockType == BlockType.STRIPED) {
|
if (blockType == BlockType.STRIPED) {
|
||||||
ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.getErasureCodingPolicy(
|
ErasureCodingPolicy ecPolicy =
|
||||||
fsd.getFSNamesystem(), inodesInPath);
|
FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy(
|
||||||
|
fsd.getFSNamesystem(), inodesInPath);
|
||||||
short numDataUnits = (short) ecPolicy.getNumDataUnits();
|
short numDataUnits = (short) ecPolicy.getNumDataUnits();
|
||||||
short numParityUnits = (short) ecPolicy.getNumParityUnits();
|
short numParityUnits = (short) ecPolicy.getNumParityUnits();
|
||||||
short numLocations = (short) (numDataUnits + numParityUnits);
|
short numLocations = (short) (numDataUnits + numParityUnits);
|
||||||
@ -529,7 +531,7 @@ private static INodesInPath addFile(
|
|||||||
try {
|
try {
|
||||||
boolean isStriped = false;
|
boolean isStriped = false;
|
||||||
ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.
|
ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.
|
||||||
getErasureCodingPolicy(fsd.getFSNamesystem(), existing);
|
unprotectedGetErasureCodingPolicy(fsd.getFSNamesystem(), existing);
|
||||||
if (ecPolicy != null) {
|
if (ecPolicy != null) {
|
||||||
isStriped = true;
|
isStriped = true;
|
||||||
}
|
}
|
||||||
|
@ -957,7 +957,8 @@ public void updateSpaceForCompleteBlock(BlockInfo completeBlk,
|
|||||||
final short replicationFactor;
|
final short replicationFactor;
|
||||||
if (fileINode.isStriped()) {
|
if (fileINode.isStriped()) {
|
||||||
final ErasureCodingPolicy ecPolicy =
|
final ErasureCodingPolicy ecPolicy =
|
||||||
FSDirErasureCodingOp.getErasureCodingPolicy(namesystem, iip);
|
FSDirErasureCodingOp
|
||||||
|
.unprotectedGetErasureCodingPolicy(namesystem, iip);
|
||||||
final short numDataUnits = (short) ecPolicy.getNumDataUnits();
|
final short numDataUnits = (short) ecPolicy.getNumDataUnits();
|
||||||
final short numParityUnits = (short) ecPolicy.getNumParityUnits();
|
final short numParityUnits = (short) ecPolicy.getNumParityUnits();
|
||||||
|
|
||||||
|
@ -415,8 +415,9 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
|
|||||||
// Update the salient file attributes.
|
// Update the salient file attributes.
|
||||||
newFile.setAccessTime(addCloseOp.atime, Snapshot.CURRENT_STATE_ID);
|
newFile.setAccessTime(addCloseOp.atime, Snapshot.CURRENT_STATE_ID);
|
||||||
newFile.setModificationTime(addCloseOp.mtime, Snapshot.CURRENT_STATE_ID);
|
newFile.setModificationTime(addCloseOp.mtime, Snapshot.CURRENT_STATE_ID);
|
||||||
ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.getErasureCodingPolicy(
|
ErasureCodingPolicy ecPolicy =
|
||||||
fsDir.getFSNamesystem(), iip);
|
FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy(
|
||||||
|
fsDir.getFSNamesystem(), iip);
|
||||||
updateBlocks(fsDir, addCloseOp, iip, newFile, ecPolicy);
|
updateBlocks(fsDir, addCloseOp, iip, newFile, ecPolicy);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@ -437,8 +438,9 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
|
|||||||
// Update the salient file attributes.
|
// Update the salient file attributes.
|
||||||
file.setAccessTime(addCloseOp.atime, Snapshot.CURRENT_STATE_ID);
|
file.setAccessTime(addCloseOp.atime, Snapshot.CURRENT_STATE_ID);
|
||||||
file.setModificationTime(addCloseOp.mtime, Snapshot.CURRENT_STATE_ID);
|
file.setModificationTime(addCloseOp.mtime, Snapshot.CURRENT_STATE_ID);
|
||||||
ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.getErasureCodingPolicy(
|
ErasureCodingPolicy ecPolicy =
|
||||||
fsDir.getFSNamesystem(), iip);
|
FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy(
|
||||||
|
fsDir.getFSNamesystem(), iip);
|
||||||
updateBlocks(fsDir, addCloseOp, iip, file, ecPolicy);
|
updateBlocks(fsDir, addCloseOp, iip, file, ecPolicy);
|
||||||
|
|
||||||
// Now close the file
|
// Now close the file
|
||||||
@ -496,8 +498,9 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
|
|||||||
INodesInPath iip = fsDir.getINodesInPath(path, DirOp.READ);
|
INodesInPath iip = fsDir.getINodesInPath(path, DirOp.READ);
|
||||||
INodeFile oldFile = INodeFile.valueOf(iip.getLastINode(), path);
|
INodeFile oldFile = INodeFile.valueOf(iip.getLastINode(), path);
|
||||||
// Update in-memory data structures
|
// Update in-memory data structures
|
||||||
ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.getErasureCodingPolicy(
|
ErasureCodingPolicy ecPolicy =
|
||||||
fsDir.getFSNamesystem(), iip);
|
FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy(
|
||||||
|
fsDir.getFSNamesystem(), iip);
|
||||||
updateBlocks(fsDir, updateOp, iip, oldFile, ecPolicy);
|
updateBlocks(fsDir, updateOp, iip, oldFile, ecPolicy);
|
||||||
|
|
||||||
if (toAddRetryCache) {
|
if (toAddRetryCache) {
|
||||||
@ -515,8 +518,9 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
|
|||||||
INodesInPath iip = fsDir.getINodesInPath(path, DirOp.READ);
|
INodesInPath iip = fsDir.getINodesInPath(path, DirOp.READ);
|
||||||
INodeFile oldFile = INodeFile.valueOf(iip.getLastINode(), path);
|
INodeFile oldFile = INodeFile.valueOf(iip.getLastINode(), path);
|
||||||
// add the new block to the INodeFile
|
// add the new block to the INodeFile
|
||||||
ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp.getErasureCodingPolicy(
|
ErasureCodingPolicy ecPolicy =
|
||||||
fsDir.getFSNamesystem(), iip);
|
FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy(
|
||||||
|
fsDir.getFSNamesystem(), iip);
|
||||||
addNewBlock(addBlockOp, oldFile, ecPolicy);
|
addNewBlock(addBlockOp, oldFile, ecPolicy);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
@ -6782,16 +6782,16 @@ void setErasureCodingPolicy(final String srcArg, final String ecPolicyName,
|
|||||||
final boolean logRetryCache) throws IOException,
|
final boolean logRetryCache) throws IOException,
|
||||||
UnresolvedLinkException, SafeModeException, AccessControlException {
|
UnresolvedLinkException, SafeModeException, AccessControlException {
|
||||||
final String operationName = "setErasureCodingPolicy";
|
final String operationName = "setErasureCodingPolicy";
|
||||||
checkSuperuserPrivilege();
|
|
||||||
checkOperation(OperationCategory.WRITE);
|
checkOperation(OperationCategory.WRITE);
|
||||||
HdfsFileStatus resultingStat = null;
|
HdfsFileStatus resultingStat = null;
|
||||||
|
final FSPermissionChecker pc = getPermissionChecker();
|
||||||
boolean success = false;
|
boolean success = false;
|
||||||
writeLock();
|
writeLock();
|
||||||
try {
|
try {
|
||||||
checkOperation(OperationCategory.WRITE);
|
checkOperation(OperationCategory.WRITE);
|
||||||
checkNameNodeSafeMode("Cannot set erasure coding policy on " + srcArg);
|
checkNameNodeSafeMode("Cannot set erasure coding policy on " + srcArg);
|
||||||
resultingStat = FSDirErasureCodingOp.setErasureCodingPolicy(this,
|
resultingStat = FSDirErasureCodingOp.setErasureCodingPolicy(this,
|
||||||
srcArg, ecPolicyName, logRetryCache);
|
srcArg, ecPolicyName, pc, logRetryCache);
|
||||||
success = true;
|
success = true;
|
||||||
} catch (AccessControlException ace) {
|
} catch (AccessControlException ace) {
|
||||||
logAuditEvent(success, operationName, srcArg, null,
|
logAuditEvent(success, operationName, srcArg, null,
|
||||||
@ -6818,16 +6818,16 @@ void unsetErasureCodingPolicy(final String srcArg,
|
|||||||
final boolean logRetryCache) throws IOException,
|
final boolean logRetryCache) throws IOException,
|
||||||
UnresolvedLinkException, SafeModeException, AccessControlException {
|
UnresolvedLinkException, SafeModeException, AccessControlException {
|
||||||
final String operationName = "unsetErasureCodingPolicy";
|
final String operationName = "unsetErasureCodingPolicy";
|
||||||
checkSuperuserPrivilege();
|
|
||||||
checkOperation(OperationCategory.WRITE);
|
checkOperation(OperationCategory.WRITE);
|
||||||
HdfsFileStatus resultingStat = null;
|
HdfsFileStatus resultingStat = null;
|
||||||
|
final FSPermissionChecker pc = getPermissionChecker();
|
||||||
boolean success = false;
|
boolean success = false;
|
||||||
writeLock();
|
writeLock();
|
||||||
try {
|
try {
|
||||||
checkOperation(OperationCategory.WRITE);
|
checkOperation(OperationCategory.WRITE);
|
||||||
checkNameNodeSafeMode("Cannot unset erasure coding policy on " + srcArg);
|
checkNameNodeSafeMode("Cannot unset erasure coding policy on " + srcArg);
|
||||||
resultingStat = FSDirErasureCodingOp.unsetErasureCodingPolicy(this,
|
resultingStat = FSDirErasureCodingOp.unsetErasureCodingPolicy(this,
|
||||||
srcArg, logRetryCache);
|
srcArg, pc, logRetryCache);
|
||||||
success = true;
|
success = true;
|
||||||
} catch (AccessControlException ace) {
|
} catch (AccessControlException ace) {
|
||||||
logAuditEvent(success, operationName, srcArg, null,
|
logAuditEvent(success, operationName, srcArg, null,
|
||||||
@ -6849,10 +6849,11 @@ void unsetErasureCodingPolicy(final String srcArg,
|
|||||||
ErasureCodingPolicy getErasureCodingPolicy(String src)
|
ErasureCodingPolicy getErasureCodingPolicy(String src)
|
||||||
throws AccessControlException, UnresolvedLinkException, IOException {
|
throws AccessControlException, UnresolvedLinkException, IOException {
|
||||||
checkOperation(OperationCategory.READ);
|
checkOperation(OperationCategory.READ);
|
||||||
|
FSPermissionChecker pc = getPermissionChecker();
|
||||||
readLock();
|
readLock();
|
||||||
try {
|
try {
|
||||||
checkOperation(OperationCategory.READ);
|
checkOperation(OperationCategory.READ);
|
||||||
return FSDirErasureCodingOp.getErasureCodingPolicy(this, src);
|
return FSDirErasureCodingOp.getErasureCodingPolicy(this, src, pc);
|
||||||
} finally {
|
} finally {
|
||||||
readUnlock("getErasureCodingPolicy");
|
readUnlock("getErasureCodingPolicy");
|
||||||
}
|
}
|
||||||
|
@ -19,6 +19,7 @@
|
|||||||
|
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.fs.CommonConfigurationKeys;
|
import org.apache.hadoop.fs.CommonConfigurationKeys;
|
||||||
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.fs.permission.FsPermission;
|
import org.apache.hadoop.fs.permission.FsPermission;
|
||||||
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
|
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
|
||||||
@ -30,12 +31,17 @@
|
|||||||
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
|
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
|
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
|
||||||
import org.apache.hadoop.io.erasurecode.ECSchema;
|
import org.apache.hadoop.io.erasurecode.ECSchema;
|
||||||
|
import org.apache.hadoop.security.AccessControlException;
|
||||||
|
import org.apache.hadoop.security.UserGroupInformation;
|
||||||
import org.junit.After;
|
import org.junit.After;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
|
import org.junit.Rule;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
import org.junit.rules.Timeout;
|
||||||
|
|
||||||
import java.io.FileNotFoundException;
|
import java.io.FileNotFoundException;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.security.PrivilegedExceptionAction;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
||||||
|
|
||||||
@ -51,6 +57,9 @@ public class TestErasureCodingPolicies {
|
|||||||
ErasureCodingPolicyManager.getSystemDefaultPolicy();
|
ErasureCodingPolicyManager.getSystemDefaultPolicy();
|
||||||
private FSNamesystem namesystem;
|
private FSNamesystem namesystem;
|
||||||
|
|
||||||
|
@Rule
|
||||||
|
public Timeout timeout = new Timeout(60 * 1000);
|
||||||
|
|
||||||
@Before
|
@Before
|
||||||
public void setupCluster() throws IOException {
|
public void setupCluster() throws IOException {
|
||||||
conf = new HdfsConfiguration();
|
conf = new HdfsConfiguration();
|
||||||
@ -74,7 +83,7 @@ public void shutdownCluster() throws IOException {
|
|||||||
* for pre-existing files (with replicated blocks) in an EC dir, getListing
|
* for pre-existing files (with replicated blocks) in an EC dir, getListing
|
||||||
* should report them as non-ec.
|
* should report them as non-ec.
|
||||||
*/
|
*/
|
||||||
@Test(timeout=60000)
|
@Test
|
||||||
public void testReplicatedFileUnderECDir() throws IOException {
|
public void testReplicatedFileUnderECDir() throws IOException {
|
||||||
final Path dir = new Path("/ec");
|
final Path dir = new Path("/ec");
|
||||||
final Path replicatedFile = new Path(dir, "replicatedFile");
|
final Path replicatedFile = new Path(dir, "replicatedFile");
|
||||||
@ -128,7 +137,7 @@ public void testReplicatedFileUnderECDir() throws IOException {
|
|||||||
assertNotNull(files[1].getErasureCodingPolicy());
|
assertNotNull(files[1].getErasureCodingPolicy());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test(timeout = 60000)
|
@Test
|
||||||
public void testBasicSetECPolicy()
|
public void testBasicSetECPolicy()
|
||||||
throws IOException, InterruptedException {
|
throws IOException, InterruptedException {
|
||||||
final Path testDir = new Path("/ec");
|
final Path testDir = new Path("/ec");
|
||||||
@ -182,7 +191,7 @@ public void testBasicSetECPolicy()
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test(timeout = 60000)
|
@Test
|
||||||
public void testMoveValidity() throws IOException, InterruptedException {
|
public void testMoveValidity() throws IOException, InterruptedException {
|
||||||
final Path srcECDir = new Path("/srcEC");
|
final Path srcECDir = new Path("/srcEC");
|
||||||
final Path dstECDir = new Path("/dstEC");
|
final Path dstECDir = new Path("/dstEC");
|
||||||
@ -219,7 +228,7 @@ public void testMoveValidity() throws IOException, InterruptedException {
|
|||||||
fs.rename(nonECFile, dstECDir);
|
fs.rename(nonECFile, dstECDir);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test(timeout = 60000)
|
@Test
|
||||||
public void testReplication() throws IOException {
|
public void testReplication() throws IOException {
|
||||||
final Path testDir = new Path("/ec");
|
final Path testDir = new Path("/ec");
|
||||||
fs.mkdir(testDir, FsPermission.getDirDefault());
|
fs.mkdir(testDir, FsPermission.getDirDefault());
|
||||||
@ -237,7 +246,7 @@ public void testReplication() throws IOException {
|
|||||||
assertEquals(policy, fs.getErasureCodingPolicy(fooFile));
|
assertEquals(policy, fs.getErasureCodingPolicy(fooFile));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test(timeout = 60000)
|
@Test
|
||||||
public void testGetErasureCodingPolicyWithSystemDefaultECPolicy() throws Exception {
|
public void testGetErasureCodingPolicyWithSystemDefaultECPolicy() throws Exception {
|
||||||
String src = "/ec";
|
String src = "/ec";
|
||||||
final Path ecDir = new Path(src);
|
final Path ecDir = new Path(src);
|
||||||
@ -254,7 +263,7 @@ public void testGetErasureCodingPolicyWithSystemDefaultECPolicy() throws Excepti
|
|||||||
verifyErasureCodingInfo(src + "/child1", sysDefaultECPolicy);
|
verifyErasureCodingInfo(src + "/child1", sysDefaultECPolicy);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test(timeout = 60000)
|
@Test
|
||||||
public void testGetErasureCodingPolicy() throws Exception {
|
public void testGetErasureCodingPolicy() throws Exception {
|
||||||
ErasureCodingPolicy[] sysECPolicies =
|
ErasureCodingPolicy[] sysECPolicies =
|
||||||
ErasureCodingPolicyManager.getSystemPolicies();
|
ErasureCodingPolicyManager.getSystemPolicies();
|
||||||
@ -284,13 +293,13 @@ private void verifyErasureCodingInfo(
|
|||||||
usingECPolicy, ecPolicy);
|
usingECPolicy, ecPolicy);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test(timeout = 60000)
|
@Test
|
||||||
public void testCreationErasureCodingZoneWithInvalidPolicy()
|
public void testSetInvalidPolicy()
|
||||||
throws IOException {
|
throws IOException {
|
||||||
ECSchema rsSchema = new ECSchema("rs", 4, 2);
|
ECSchema rsSchema = new ECSchema("rs", 4, 2);
|
||||||
String policyName = "RS-4-2-128k";
|
String policyName = "RS-4-2-128k";
|
||||||
int cellSize = 128 * 1024;
|
int cellSize = 128 * 1024;
|
||||||
ErasureCodingPolicy ecPolicy=
|
ErasureCodingPolicy ecPolicy =
|
||||||
new ErasureCodingPolicy(policyName, rsSchema, cellSize, (byte) -1);
|
new ErasureCodingPolicy(policyName, rsSchema, cellSize, (byte) -1);
|
||||||
String src = "/ecDir4-2";
|
String src = "/ecDir4-2";
|
||||||
final Path ecDir = new Path(src);
|
final Path ecDir = new Path(src);
|
||||||
@ -305,7 +314,7 @@ public void testCreationErasureCodingZoneWithInvalidPolicy()
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test(timeout = 60000)
|
@Test
|
||||||
public void testGetAllErasureCodingPolicies() throws Exception {
|
public void testGetAllErasureCodingPolicies() throws Exception {
|
||||||
ErasureCodingPolicy[] sysECPolicies = ErasureCodingPolicyManager
|
ErasureCodingPolicy[] sysECPolicies = ErasureCodingPolicyManager
|
||||||
.getSystemPolicies();
|
.getSystemPolicies();
|
||||||
@ -315,7 +324,7 @@ public void testGetAllErasureCodingPolicies() throws Exception {
|
|||||||
allECPolicies.containsAll(Arrays.asList(sysECPolicies)));
|
allECPolicies.containsAll(Arrays.asList(sysECPolicies)));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test(timeout = 60000)
|
@Test
|
||||||
public void testGetErasureCodingPolicyOnANonExistentFile() throws Exception {
|
public void testGetErasureCodingPolicyOnANonExistentFile() throws Exception {
|
||||||
Path path = new Path("/ecDir");
|
Path path = new Path("/ecDir");
|
||||||
try {
|
try {
|
||||||
@ -335,7 +344,7 @@ public void testGetErasureCodingPolicyOnANonExistentFile() throws Exception {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test(timeout = 60000)
|
@Test
|
||||||
public void testMultiplePoliciesCoExist() throws Exception {
|
public void testMultiplePoliciesCoExist() throws Exception {
|
||||||
ErasureCodingPolicy[] sysPolicies =
|
ErasureCodingPolicy[] sysPolicies =
|
||||||
ErasureCodingPolicyManager.getSystemPolicies();
|
ErasureCodingPolicyManager.getSystemPolicies();
|
||||||
@ -355,4 +364,124 @@ public void testMultiplePoliciesCoExist() throws Exception {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testPermissions() throws Exception {
|
||||||
|
UserGroupInformation user =
|
||||||
|
UserGroupInformation.createUserForTesting("ecuser",
|
||||||
|
new String[]{"ecgroup"});
|
||||||
|
FileSystem userfs = user.doAs(new PrivilegedExceptionAction<FileSystem>() {
|
||||||
|
@Override
|
||||||
|
public FileSystem run() throws Exception {
|
||||||
|
return FileSystem.get(conf);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
HdfsAdmin useradmin = user.doAs(new PrivilegedExceptionAction<HdfsAdmin>() {
|
||||||
|
@Override
|
||||||
|
public HdfsAdmin run() throws Exception {
|
||||||
|
return new HdfsAdmin(userfs.getUri(), conf);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Create dir and set an EC policy, create an EC file
|
||||||
|
Path ecdir = new Path("/ecdir");
|
||||||
|
Path ecfile = new Path(ecdir, "ecfile");
|
||||||
|
fs.setPermission(new Path("/"), new FsPermission((short)0777));
|
||||||
|
userfs.mkdirs(ecdir);
|
||||||
|
final String ecPolicyName =
|
||||||
|
ErasureCodingPolicyManager.getSystemPolicies()[0].getName();
|
||||||
|
useradmin.setErasureCodingPolicy(ecdir, ecPolicyName);
|
||||||
|
assertEquals("Policy not present on dir",
|
||||||
|
ecPolicyName,
|
||||||
|
useradmin.getErasureCodingPolicy(ecdir).getName());
|
||||||
|
userfs.create(ecfile).close();
|
||||||
|
assertEquals("Policy not present on file",
|
||||||
|
ecPolicyName,
|
||||||
|
useradmin.getErasureCodingPolicy(ecfile).getName());
|
||||||
|
|
||||||
|
// Unset and re-set
|
||||||
|
useradmin.unsetErasureCodingPolicy(ecdir);
|
||||||
|
useradmin.setErasureCodingPolicy(ecdir, ecPolicyName);
|
||||||
|
|
||||||
|
// Change write permissions and make sure set and unset are denied
|
||||||
|
userfs.setPermission(ecdir, new FsPermission((short)0555));
|
||||||
|
try {
|
||||||
|
useradmin.setErasureCodingPolicy(ecdir, ecPolicyName);
|
||||||
|
fail("Should not be able to setECPolicy without write permissions");
|
||||||
|
} catch (AccessControlException e) {
|
||||||
|
// pass
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
useradmin.unsetErasureCodingPolicy(ecdir);
|
||||||
|
fail("Should not be able to unsetECPolicy without write permissions");
|
||||||
|
} catch (AccessControlException e) {
|
||||||
|
// pass
|
||||||
|
}
|
||||||
|
|
||||||
|
// Change the permissions again, check that set and unset work
|
||||||
|
userfs.setPermission(ecdir, new FsPermission((short)0640));
|
||||||
|
useradmin.unsetErasureCodingPolicy(ecdir);
|
||||||
|
useradmin.setErasureCodingPolicy(ecdir, ecPolicyName);
|
||||||
|
|
||||||
|
// Set, unset, and get with another user should be unauthorized
|
||||||
|
UserGroupInformation nobody =
|
||||||
|
UserGroupInformation.createUserForTesting("nobody",
|
||||||
|
new String[]{"nogroup"});
|
||||||
|
HdfsAdmin noadmin = nobody.doAs(new PrivilegedExceptionAction<HdfsAdmin>() {
|
||||||
|
@Override
|
||||||
|
public HdfsAdmin run() throws Exception {
|
||||||
|
return new HdfsAdmin(userfs.getUri(), conf);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
noadmin.setErasureCodingPolicy(ecdir, ecPolicyName);
|
||||||
|
fail("Should not be able to setECPolicy without write permissions");
|
||||||
|
} catch (AccessControlException e) {
|
||||||
|
// pass
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
noadmin.unsetErasureCodingPolicy(ecdir);
|
||||||
|
fail("Should not be able to unsetECPolicy without write permissions");
|
||||||
|
} catch (AccessControlException e) {
|
||||||
|
// pass
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
noadmin.getErasureCodingPolicy(ecdir);
|
||||||
|
fail("Should not be able to getECPolicy without write permissions");
|
||||||
|
} catch (AccessControlException e) {
|
||||||
|
// pass
|
||||||
|
}
|
||||||
|
|
||||||
|
// superuser can do whatever it wants
|
||||||
|
userfs.setPermission(ecdir, new FsPermission((short)0000));
|
||||||
|
HdfsAdmin superadmin = new HdfsAdmin(fs.getUri(), conf);
|
||||||
|
superadmin.unsetErasureCodingPolicy(ecdir);
|
||||||
|
superadmin.setErasureCodingPolicy(ecdir, ecPolicyName);
|
||||||
|
superadmin.getErasureCodingPolicy(ecdir);
|
||||||
|
|
||||||
|
// Normal user no longer has access
|
||||||
|
try {
|
||||||
|
useradmin.getErasureCodingPolicy(ecdir);
|
||||||
|
fail("Normal user should not have access");
|
||||||
|
} catch (AccessControlException e) {
|
||||||
|
// pass
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
useradmin.setErasureCodingPolicy(ecfile, ecPolicyName);
|
||||||
|
fail("Normal user should not have access");
|
||||||
|
} catch (AccessControlException e) {
|
||||||
|
// pass
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
useradmin.unsetErasureCodingPolicy(ecfile);
|
||||||
|
fail("Normal user should not have access");
|
||||||
|
} catch (AccessControlException e) {
|
||||||
|
// pass
|
||||||
|
}
|
||||||
|
|
||||||
|
// Everyone has access to getting the list of EC policies
|
||||||
|
useradmin.getErasureCodingPolicies();
|
||||||
|
noadmin.getErasureCodingPolicies();
|
||||||
|
superadmin.getErasureCodingPolicies();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user