big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-7486. Consolidate XAttr-related implementation into a single class. Contributed by Haohui Mai.

Browse Source
This commit is contained in:
Haohui Mai 2014-12-08 11:52:21 -08:00
parent 57cb43be50
commit 6c5bbd7a42
7 changed files with 554 additions and 500 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -444,6 +444,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7384. 'getfacl' command and 'getAclStatus' output should be in sync.
(Vinayakumar B via cnauroth)
HDFS-7486. Consolidate XAttr-related implementation into a single class.
(wheat9)
OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.

3
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
View File

@ -311,7 +311,8 @@ XAttr createEncryptionZone(String src, CipherSuite suite,
xattrs.add(ezXAttr);
// updating the xattr will call addEncryptionZone,
// done this way to handle edit log loading
dir.unprotectedSetXAttrs(src, xattrs, EnumSet.of(XAttrSetFlag.CREATE));
FSDirXAttrOp.unprotectedSetXAttrs(dir, src, xattrs,
EnumSet.of(XAttrSetFlag.CREATE));
return ezXAttr;
}

460
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java Normal file
View File

@ -0,0 +1,460 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Charsets;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.XAttrHelper;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
import org.apache.hadoop.hdfs.protocolPB.PBHelper;
import org.apache.hadoop.security.AccessControlException;
import java.io.IOException;
import java.util.EnumSet;
import java.util.List;
import java.util.ListIterator;
import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_ENCRYPTION_ZONE;
import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.SECURITY_XATTR_UNREADABLE_BY_SUPERUSER;
class FSDirXAttrOp {
private static final XAttr KEYID_XATTR =
XAttrHelper.buildXAttr(CRYPTO_XATTR_ENCRYPTION_ZONE, null);
private static final XAttr UNREADABLE_BY_SUPERUSER_XATTR =
XAttrHelper.buildXAttr(SECURITY_XATTR_UNREADABLE_BY_SUPERUSER, null);
/**
* Set xattr for a file or directory.
*
* @param src
* - path on which it sets the xattr
* @param xAttr
* - xAttr details to set
* @param flag
* - xAttrs flags
* @throws IOException
*/
static HdfsFileStatus setXAttr(
FSDirectory fsd, String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag,
boolean logRetryCache)
throws IOException {
checkXAttrsConfigFlag(fsd);
checkXAttrSize(fsd, xAttr);
FSPermissionChecker pc = fsd.getPermissionChecker();
XAttrPermissionFilter.checkPermissionForApi(
pc, xAttr, FSDirectory.isReservedRawName(src));
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(
src);
src = fsd.resolvePath(pc, src, pathComponents);
final INodesInPath iip = fsd.getINodesInPath4Write(src);
checkXAttrChangeAccess(fsd, iip, xAttr, pc);
List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
xAttrs.add(xAttr);
fsd.writeLock();
try {
unprotectedSetXAttrs(fsd, src, xAttrs, flag);
} finally {
fsd.writeUnlock();
}
fsd.getEditLog().logSetXAttrs(src, xAttrs, logRetryCache);
return fsd.getAuditFileInfo(src, false);
}
static List<XAttr> getXAttrs(FSDirectory fsd, final String srcArg,
List<XAttr> xAttrs)
throws IOException {
String src = srcArg;
checkXAttrsConfigFlag(fsd);
FSPermissionChecker pc = fsd.getPermissionChecker();
final boolean isRawPath = FSDirectory.isReservedRawName(src);
boolean getAll = xAttrs == null || xAttrs.isEmpty();
if (!getAll) {
XAttrPermissionFilter.checkPermissionForApi(pc, xAttrs, isRawPath);
}
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
src = fsd.resolvePath(pc, src, pathComponents);
final INodesInPath iip = fsd.getINodesInPath(src, true);
if (fsd.isPermissionEnabled()) {
fsd.checkPathAccess(pc, iip, FsAction.READ);
}
List<XAttr> all = FSDirXAttrOp.getXAttrs(fsd, src);
List<XAttr> filteredAll = XAttrPermissionFilter.
filterXAttrsForApi(pc, all, isRawPath);
if (getAll) {
return filteredAll;
}
if (filteredAll == null || filteredAll.isEmpty()) {
return null;
}
List<XAttr> toGet = Lists.newArrayListWithCapacity(xAttrs.size());
for (XAttr xAttr : xAttrs) {
boolean foundIt = false;
for (XAttr a : filteredAll) {
if (xAttr.getNameSpace() == a.getNameSpace() && xAttr.getName().equals(
a.getName())) {
toGet.add(a);
foundIt = true;
break;
}
}
if (!foundIt) {
throw new IOException(
"At least one of the attributes provided was not found.");
}
}
return toGet;
}
static List<XAttr> listXAttrs(
FSDirectory fsd, String src) throws IOException {
FSDirXAttrOp.checkXAttrsConfigFlag(fsd);
final FSPermissionChecker pc = fsd.getPermissionChecker();
final boolean isRawPath = FSDirectory.isReservedRawName(src);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
src = fsd.resolvePath(pc, src, pathComponents);
final INodesInPath iip = fsd.getINodesInPath(src, true);
if (fsd.isPermissionEnabled()) {
/* To access xattr names, you need EXECUTE in the owning directory. */
fsd.checkParentAccess(pc, iip, FsAction.EXECUTE);
}
final List<XAttr> all = FSDirXAttrOp.getXAttrs(fsd, src);
return XAttrPermissionFilter.
filterXAttrsForApi(pc, all, isRawPath);
}
/**
* Remove an xattr for a file or directory.
*
* @param src
* - path to remove the xattr from
* @param xAttr
* - xAttr to remove
* @throws IOException
*/
static HdfsFileStatus removeXAttr(
FSDirectory fsd, String src, XAttr xAttr, boolean logRetryCache)
throws IOException {
FSDirXAttrOp.checkXAttrsConfigFlag(fsd);
FSPermissionChecker pc = fsd.getPermissionChecker();
XAttrPermissionFilter.checkPermissionForApi(
pc, xAttr, FSDirectory.isReservedRawName(src));
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(
src);
src = fsd.resolvePath(pc, src, pathComponents);
final INodesInPath iip = fsd.getINodesInPath4Write(src);
checkXAttrChangeAccess(fsd, iip, xAttr, pc);
List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
xAttrs.add(xAttr);
fsd.writeLock();
try {
List<XAttr> removedXAttrs = unprotectedRemoveXAttrs(fsd, src, xAttrs);
if (removedXAttrs != null && !removedXAttrs.isEmpty()) {
fsd.getEditLog().logRemoveXAttrs(src, removedXAttrs, logRetryCache);
} else {
throw new IOException(
"No matching attributes found for remove operation");
}
} finally {
fsd.writeUnlock();
}
return fsd.getAuditFileInfo(src, false);
}
static List<XAttr> unprotectedRemoveXAttrs(
FSDirectory fsd, final String src, final List<XAttr> toRemove)
throws IOException {
assert fsd.hasWriteLock();
INodesInPath iip = fsd.getINodesInPath4Write(
FSDirectory.normalizePath(src), true);
INode inode = FSDirectory.resolveLastINode(src, iip);
int snapshotId = iip.getLatestSnapshotId();
List<XAttr> existingXAttrs = XAttrStorage.readINodeXAttrs(inode);
List<XAttr> removedXAttrs = Lists.newArrayListWithCapacity(toRemove.size());
List<XAttr> newXAttrs = filterINodeXAttrs(existingXAttrs, toRemove,
removedXAttrs);
if (existingXAttrs.size() != newXAttrs.size()) {
XAttrStorage.updateINodeXAttrs(inode, newXAttrs, snapshotId);
return removedXAttrs;
}
return null;
}
/**
* Filter XAttrs from a list of existing XAttrs. Removes matched XAttrs from
* toFilter and puts them into filtered. Upon completion,
* toFilter contains the filter XAttrs that were not found, while
* fitleredXAttrs contains the XAttrs that were found.
*
* @param existingXAttrs Existing XAttrs to be filtered
* @param toFilter XAttrs to filter from the existing XAttrs
* @param filtered Return parameter, XAttrs that were filtered
* @return List of XAttrs that does not contain filtered XAttrs
*/
@VisibleForTesting
static List<XAttr> filterINodeXAttrs(
final List<XAttr> existingXAttrs, final List<XAttr> toFilter,
final List<XAttr> filtered)
throws AccessControlException {
if (existingXAttrs == null || existingXAttrs.isEmpty() ||
toFilter == null || toFilter.isEmpty()) {
return existingXAttrs;
}
// Populate a new list with XAttrs that pass the filter
List<XAttr> newXAttrs =
Lists.newArrayListWithCapacity(existingXAttrs.size());
for (XAttr a : existingXAttrs) {
boolean add = true;
for (ListIterator<XAttr> it = toFilter.listIterator(); it.hasNext()
;) {
XAttr filter = it.next();
Preconditions.checkArgument(
!KEYID_XATTR.equalsIgnoreValue(filter),
"The encryption zone xattr should never be deleted.");
if (UNREADABLE_BY_SUPERUSER_XATTR.equalsIgnoreValue(filter)) {
throw new AccessControlException("The xattr '" +
SECURITY_XATTR_UNREADABLE_BY_SUPERUSER + "' can not be deleted.");
}
if (a.equalsIgnoreValue(filter)) {
add = false;
it.remove();
filtered.add(filter);
break;
}
}
if (add) {
newXAttrs.add(a);
}
}
return newXAttrs;
}
static INode unprotectedSetXAttrs(
FSDirectory fsd, final String src, final List<XAttr> xAttrs,
final EnumSet<XAttrSetFlag> flag)
throws IOException {
assert fsd.hasWriteLock();
INodesInPath iip = fsd.getINodesInPath4Write(FSDirectory.normalizePath(src), true);
INode inode = FSDirectory.resolveLastINode(src, iip);
int snapshotId = iip.getLatestSnapshotId();
List<XAttr> existingXAttrs = XAttrStorage.readINodeXAttrs(inode);
List<XAttr> newXAttrs = setINodeXAttrs(fsd, existingXAttrs, xAttrs, flag);
final boolean isFile = inode.isFile();
for (XAttr xattr : newXAttrs) {
final String xaName = XAttrHelper.getPrefixName(xattr);
/*
* If we're adding the encryption zone xattr, then add src to the list
* of encryption zones.
*/
if (CRYPTO_XATTR_ENCRYPTION_ZONE.equals(xaName)) {
final HdfsProtos.ZoneEncryptionInfoProto ezProto =
HdfsProtos.ZoneEncryptionInfoProto.parseFrom(xattr.getValue());
fsd.ezManager.addEncryptionZone(inode.getId(),
PBHelper.convert(ezProto.getSuite()),
PBHelper.convert(
ezProto.getCryptoProtocolVersion()),
ezProto.getKeyName());
}
if (!isFile && SECURITY_XATTR_UNREADABLE_BY_SUPERUSER.equals(xaName)) {
throw new IOException("Can only set '" +
SECURITY_XATTR_UNREADABLE_BY_SUPERUSER + "' on a file.");
}
}
XAttrStorage.updateINodeXAttrs(inode, newXAttrs, snapshotId);
return inode;
}
static List<XAttr> setINodeXAttrs(
FSDirectory fsd, final List<XAttr> existingXAttrs,
final List<XAttr> toSet, final EnumSet<XAttrSetFlag> flag)
throws IOException {
// Check for duplicate XAttrs in toSet
// We need to use a custom comparator, so using a HashSet is not suitable
for (int i = 0; i < toSet.size(); i++) {
for (int j = i + 1; j < toSet.size(); j++) {
if (toSet.get(i).equalsIgnoreValue(toSet.get(j))) {
throw new IOException("Cannot specify the same XAttr to be set " +
"more than once");
}
}
}
// Count the current number of user-visible XAttrs for limit checking
int userVisibleXAttrsNum = 0; // Number of user visible xAttrs
// The XAttr list is copied to an exactly-sized array when it's stored,
// so there's no need to size it precisely here.
int newSize = (existingXAttrs != null) ? existingXAttrs.size() : 0;
newSize += toSet.size();
List<XAttr> xAttrs = Lists.newArrayListWithCapacity(newSize);
// Check if the XAttr already exists to validate with the provided flag
for (XAttr xAttr: toSet) {
boolean exist = false;
if (existingXAttrs != null) {
for (XAttr a : existingXAttrs) {
if (a.equalsIgnoreValue(xAttr)) {
exist = true;
break;
}
}
}
XAttrSetFlag.validate(xAttr.getName(), exist, flag);
// add the new XAttr since it passed validation
xAttrs.add(xAttr);
if (isUserVisible(xAttr)) {
userVisibleXAttrsNum++;
}
}
// Add the existing xattrs back in, if they weren't already set
if (existingXAttrs != null) {
for (XAttr existing : existingXAttrs) {
boolean alreadySet = false;
for (XAttr set : toSet) {
if (set.equalsIgnoreValue(existing)) {
alreadySet = true;
break;
}
}
if (!alreadySet) {
xAttrs.add(existing);
if (isUserVisible(existing)) {
userVisibleXAttrsNum++;
}
}
}
}
if (userVisibleXAttrsNum > fsd.getInodeXAttrsLimit()) {
throw new IOException("Cannot add additional XAttr to inode, "
+ "would exceed limit of " + fsd.getInodeXAttrsLimit());
}
return xAttrs;
}
static List<XAttr> getXAttrs(FSDirectory fsd, INode inode, int snapshotId)
throws IOException {
fsd.readLock();
try {
return XAttrStorage.readINodeXAttrs(inode, snapshotId);
} finally {
fsd.readUnlock();
}
}
static XAttr unprotectedGetXAttrByName(
INode inode, int snapshotId, String xAttrName)
throws IOException {
List<XAttr> xAttrs = XAttrStorage.readINodeXAttrs(inode, snapshotId);
if (xAttrs == null) {
return null;
}
for (XAttr x : xAttrs) {
if (XAttrHelper.getPrefixName(x)
.equals(xAttrName)) {
return x;
}
}
return null;
}
private static void checkXAttrChangeAccess(
FSDirectory fsd, INodesInPath iip, XAttr xAttr,
FSPermissionChecker pc)
throws AccessControlException {
if (fsd.isPermissionEnabled() && xAttr.getNameSpace() == XAttr.NameSpace
.USER) {
final INode inode = iip.getLastINode();
if (inode != null &&
inode.isDirectory() &&
inode.getFsPermission().getStickyBit()) {
if (!pc.isSuperUser()) {
fsd.checkOwner(pc, iip);
}
} else {
fsd.checkPathAccess(pc, iip, FsAction.WRITE);
}
}
}
/**
* Verifies that the combined size of the name and value of an xattr is within
* the configured limit. Setting a limit of zero disables this check.
*/
private static void checkXAttrSize(FSDirectory fsd, XAttr xAttr) {
if (fsd.getXattrMaxSize() == 0) {
return;
}
int size = xAttr.getName().getBytes(Charsets.UTF_8).length;
if (xAttr.getValue() != null) {
size += xAttr.getValue().length;
}
if (size > fsd.getXattrMaxSize()) {
throw new HadoopIllegalArgumentException(
"The XAttr is too big. The maximum combined size of the"
+ " name and value is " + fsd.getXattrMaxSize()
+ ", but the total size is " + size);
}
}
private static void checkXAttrsConfigFlag(FSDirectory fsd) throws
IOException {
if (!fsd.isXattrsEnabled()) {
throw new IOException(String.format(
"The XAttr operation has been rejected. "
+ "Support for XAttrs has been disabled by setting %s to false.",
DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY));
}
}
private static List<XAttr> getXAttrs(FSDirectory fsd,
String src) throws IOException {
String srcs = FSDirectory.normalizePath(src);
fsd.readLock();
try {
INodesInPath iip = fsd.getLastINodeInPath(srcs, true);
INode inode = FSDirectory.resolveLastINode(src, iip);
int snapshotId = iip.getPathSnapshotId();
return XAttrStorage.readINodeXAttrs(inode, snapshotId);
} finally {
fsd.readUnlock();
}
}
private static boolean isUserVisible(XAttr xAttr) {
XAttr.NameSpace ns = xAttr.getNameSpace();
return ns == XAttr.NameSpace.USER || ns == XAttr.NameSpace.TRUSTED;
}
}

295
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
View File

@ -20,7 +20,6 @@
import static org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries;
import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_ENCRYPTION_ZONE;
import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_FILE_ENCRYPTION_INFO;
import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.SECURITY_XATTR_UNREADABLE_BY_SUPERUSER;
import static org.apache.hadoop.util.Time.now;
import java.io.Closeable;
@ -30,7 +29,6 @@
import java.util.Arrays;
import java.util.EnumSet;
import java.util.List;
import java.util.ListIterator;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import com.google.protobuf.InvalidProtocolBufferException;
@ -122,10 +120,6 @@ private static INodeDirectory createRoot(FSNamesystem namesystem) {
public final static String DOT_INODES_STRING = ".inodes";
public final static byte[] DOT_INODES =
DFSUtil.string2Bytes(DOT_INODES_STRING);
private final XAttr KEYID_XATTR =
XAttrHelper.buildXAttr(CRYPTO_XATTR_ENCRYPTION_ZONE, null);
private final XAttr UNREADABLE_BY_SUPERUSER_XATTR =
XAttrHelper.buildXAttr(SECURITY_XATTR_UNREADABLE_BY_SUPERUSER, null);
INodeDirectory rootDir;
private final FSNamesystem namesystem;
@ -136,6 +130,7 @@ private static INodeDirectory createRoot(FSNamesystem namesystem) {
private final int contentCountLimit; // max content summary counts per run
private final INodeMap inodeMap; // Synchronized by dirLock
private long yieldCount = 0; // keep track of lock yield count.
private final int inodeXAttrsLimit; //inode xattrs max limit
// lock to protect the directory and BlockMap
@ -148,6 +143,8 @@ private static INodeDirectory createRoot(FSNamesystem namesystem) {
* ACL-related operations.
*/
private final boolean aclsEnabled;
private final boolean xattrsEnabled;
private final int xattrMaxSize;
private final String fsOwnerShortUserName;
private final String supergroup;
private final INodeId inodeId;
@ -213,6 +210,18 @@ public int getWriteHoldCount() {
DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY,
DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_DEFAULT);
LOG.info("ACLs enabled? " + aclsEnabled);
this.xattrsEnabled = conf.getBoolean(
DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY,
DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_DEFAULT);
LOG.info("XAttrs enabled? " + xattrsEnabled);
this.xattrMaxSize = conf.getInt(
DFSConfigKeys.DFS_NAMENODE_MAX_XATTR_SIZE_KEY,
DFSConfigKeys.DFS_NAMENODE_MAX_XATTR_SIZE_DEFAULT);
Preconditions.checkArgument(xattrMaxSize >= 0,
"Cannot set a negative value for the maximum size of an xattr (%s).",
DFSConfigKeys.DFS_NAMENODE_MAX_XATTR_SIZE_KEY);
final String unlimited = xattrMaxSize == 0 ? " (unlimited)" : "";
LOG.info("Maximum size of an xattr: " + xattrMaxSize + unlimited);
int configuredLimit = conf.getInt(
DFSConfigKeys.DFS_LIST_LIMIT, DFSConfigKeys.DFS_LIST_LIMIT_DEFAULT);
this.lsLimit = configuredLimit>0 ?
@ -274,6 +283,10 @@ boolean isPermissionEnabled() {
boolean isAclsEnabled() {
return aclsEnabled;
}
boolean isXattrsEnabled() {
return xattrsEnabled;
}
int getXattrMaxSize() { return xattrMaxSize; }
int getLsLimit() {
return lsLimit;
@ -283,6 +296,10 @@ int getContentCountLimit() {
return contentCountLimit;
}
int getInodeXAttrsLimit() {
return inodeXAttrsLimit;
}
FSEditLog getEditLog() {
return editLog;
}
@ -613,8 +630,11 @@ private void setDirStoragePolicy(INodeDirectory inode, byte policyId,
int latestSnapshotId) throws IOException {
List<XAttr> existingXAttrs = XAttrStorage.readINodeXAttrs(inode);
XAttr xAttr = BlockStoragePolicySuite.buildXAttr(policyId);
List<XAttr> newXAttrs = setINodeXAttrs(existingXAttrs, Arrays.asList(xAttr),
EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE));
List<XAttr> newXAttrs = FSDirXAttrOp.setINodeXAttrs(this, existingXAttrs,
Arrays.asList(xAttr),
EnumSet.of(
XAttrSetFlag.CREATE,
XAttrSetFlag.REPLACE));
XAttrStorage.updateINodeXAttrs(inode, newXAttrs, latestSnapshotId);
}
@ -1560,90 +1580,6 @@ INodeSymlink unprotectedAddSymlink(long id, String path, String target,
return addINode(path, symlink) ? symlink : null;
}
/**
* Removes a list of XAttrs from an inode at a path.
*
* @param src path of inode
* @param toRemove XAttrs to be removed
* @return List of XAttrs that were removed
* @throws IOException if the inode does not exist, if quota is exceeded
*/
List<XAttr> removeXAttrs(final String src, final List<XAttr> toRemove)
throws IOException {
writeLock();
try {
return unprotectedRemoveXAttrs(src, toRemove);
} finally {
writeUnlock();
}
}
List<XAttr> unprotectedRemoveXAttrs(final String src,
final List<XAttr> toRemove) throws IOException {
assert hasWriteLock();
INodesInPath iip = getINodesInPath4Write(normalizePath(src), true);
INode inode = resolveLastINode(src, iip);
int snapshotId = iip.getLatestSnapshotId();
List<XAttr> existingXAttrs = XAttrStorage.readINodeXAttrs(inode);
List<XAttr> removedXAttrs = Lists.newArrayListWithCapacity(toRemove.size());
List<XAttr> newXAttrs = filterINodeXAttrs(existingXAttrs, toRemove,
removedXAttrs);
if (existingXAttrs.size() != newXAttrs.size()) {
XAttrStorage.updateINodeXAttrs(inode, newXAttrs, snapshotId);
return removedXAttrs;
}
return null;
}
/**
* Filter XAttrs from a list of existing XAttrs. Removes matched XAttrs from
* toFilter and puts them into filtered. Upon completion,
* toFilter contains the filter XAttrs that were not found, while
* fitleredXAttrs contains the XAttrs that were found.
*
* @param existingXAttrs Existing XAttrs to be filtered
* @param toFilter XAttrs to filter from the existing XAttrs
* @param filtered Return parameter, XAttrs that were filtered
* @return List of XAttrs that does not contain filtered XAttrs
*/
@VisibleForTesting
List<XAttr> filterINodeXAttrs(final List<XAttr> existingXAttrs,
final List<XAttr> toFilter, final List<XAttr> filtered)
throws AccessControlException {
if (existingXAttrs == null || existingXAttrs.isEmpty() ||
toFilter == null || toFilter.isEmpty()) {
return existingXAttrs;
}
// Populate a new list with XAttrs that pass the filter
List<XAttr> newXAttrs =
Lists.newArrayListWithCapacity(existingXAttrs.size());
for (XAttr a : existingXAttrs) {
boolean add = true;
for (ListIterator<XAttr> it = toFilter.listIterator(); it.hasNext()
;) {
XAttr filter = it.next();
Preconditions.checkArgument(!KEYID_XATTR.equalsIgnoreValue(filter),
"The encryption zone xattr should never be deleted.");
if (UNREADABLE_BY_SUPERUSER_XATTR.equalsIgnoreValue(filter)) {
throw new AccessControlException("The xattr '" +
SECURITY_XATTR_UNREADABLE_BY_SUPERUSER + "' can not be deleted.");
}
if (a.equalsIgnoreValue(filter)) {
add = false;
it.remove();
filtered.add(filter);
break;
}
}
if (add) {
newXAttrs.add(a);
}
}
return newXAttrs;
}
boolean isInAnEZ(INodesInPath iip)
throws UnresolvedLinkException, SnapshotAccessControlException {
readLock();
@ -1709,7 +1645,8 @@ void setFileEncryptionInfo(String src, FileEncryptionInfo info)
writeLock();
try {
unprotectedSetXAttrs(src, xAttrs, EnumSet.of(XAttrSetFlag.CREATE));
FSDirXAttrOp.unprotectedSetXAttrs(this, src, xAttrs,
EnumSet.of(XAttrSetFlag.CREATE));
} finally {
writeUnlock();
}
@ -1752,8 +1689,9 @@ FileEncryptionInfo getFileEncryptionInfo(INode inode, int snapshotId,
final CipherSuite suite = encryptionZone.getSuite();
final String keyName = encryptionZone.getKeyName();
XAttr fileXAttr = unprotectedGetXAttrByName(inode, snapshotId,
CRYPTO_XATTR_FILE_ENCRYPTION_INFO);
XAttr fileXAttr = FSDirXAttrOp.unprotectedGetXAttrByName(inode,
snapshotId,
CRYPTO_XATTR_FILE_ENCRYPTION_INFO);
if (fileXAttr == null) {
NameNode.LOG.warn("Could not find encryption XAttr for file " +
@ -1775,173 +1713,6 @@ FileEncryptionInfo getFileEncryptionInfo(INode inode, int snapshotId,
}
}
void setXAttrs(final String src, final List<XAttr> xAttrs,
final EnumSet<XAttrSetFlag> flag) throws IOException {
writeLock();
try {
unprotectedSetXAttrs(src, xAttrs, flag);
} finally {
writeUnlock();
}
}
INode unprotectedSetXAttrs(final String src, final List<XAttr> xAttrs,
final EnumSet<XAttrSetFlag> flag)
throws QuotaExceededException, IOException {
assert hasWriteLock();
INodesInPath iip = getINodesInPath4Write(normalizePath(src), true);
INode inode = resolveLastINode(src, iip);
int snapshotId = iip.getLatestSnapshotId();
List<XAttr> existingXAttrs = XAttrStorage.readINodeXAttrs(inode);
List<XAttr> newXAttrs = setINodeXAttrs(existingXAttrs, xAttrs, flag);
final boolean isFile = inode.isFile();
for (XAttr xattr : newXAttrs) {
final String xaName = XAttrHelper.getPrefixName(xattr);
/*
* If we're adding the encryption zone xattr, then add src to the list
* of encryption zones.
*/
if (CRYPTO_XATTR_ENCRYPTION_ZONE.equals(xaName)) {
final HdfsProtos.ZoneEncryptionInfoProto ezProto =
HdfsProtos.ZoneEncryptionInfoProto.parseFrom(xattr.getValue());
ezManager.addEncryptionZone(inode.getId(),
PBHelper.convert(ezProto.getSuite()),
PBHelper.convert(ezProto.getCryptoProtocolVersion()),
ezProto.getKeyName());
}
if (!isFile && SECURITY_XATTR_UNREADABLE_BY_SUPERUSER.equals(xaName)) {
throw new IOException("Can only set '" +
SECURITY_XATTR_UNREADABLE_BY_SUPERUSER + "' on a file.");
}
}
XAttrStorage.updateINodeXAttrs(inode, newXAttrs, snapshotId);
return inode;
}
List<XAttr> setINodeXAttrs(final List<XAttr> existingXAttrs,
final List<XAttr> toSet, final EnumSet<XAttrSetFlag> flag)
throws IOException {
// Check for duplicate XAttrs in toSet
// We need to use a custom comparator, so using a HashSet is not suitable
for (int i = 0; i < toSet.size(); i++) {
for (int j = i + 1; j < toSet.size(); j++) {
if (toSet.get(i).equalsIgnoreValue(toSet.get(j))) {
throw new IOException("Cannot specify the same XAttr to be set " +
"more than once");
}
}
}
// Count the current number of user-visible XAttrs for limit checking
int userVisibleXAttrsNum = 0; // Number of user visible xAttrs
// The XAttr list is copied to an exactly-sized array when it's stored,
// so there's no need to size it precisely here.
int newSize = (existingXAttrs != null) ? existingXAttrs.size() : 0;
newSize += toSet.size();
List<XAttr> xAttrs = Lists.newArrayListWithCapacity(newSize);
// Check if the XAttr already exists to validate with the provided flag
for (XAttr xAttr: toSet) {
boolean exist = false;
if (existingXAttrs != null) {
for (XAttr a : existingXAttrs) {
if (a.equalsIgnoreValue(xAttr)) {
exist = true;
break;
}
}
}
XAttrSetFlag.validate(xAttr.getName(), exist, flag);
// add the new XAttr since it passed validation
xAttrs.add(xAttr);
if (isUserVisible(xAttr)) {
userVisibleXAttrsNum++;
}
}
// Add the existing xattrs back in, if they weren't already set
if (existingXAttrs != null) {
for (XAttr existing : existingXAttrs) {
boolean alreadySet = false;
for (XAttr set : toSet) {
if (set.equalsIgnoreValue(existing)) {
alreadySet = true;
break;
}
}
if (!alreadySet) {
xAttrs.add(existing);
if (isUserVisible(existing)) {
userVisibleXAttrsNum++;
}
}
}
}
if (userVisibleXAttrsNum > inodeXAttrsLimit) {
throw new IOException("Cannot add additional XAttr to inode, "
+ "would exceed limit of " + inodeXAttrsLimit);
}
return xAttrs;
}
private boolean isUserVisible(XAttr xAttr) {
if (xAttr.getNameSpace() == XAttr.NameSpace.USER ||
xAttr.getNameSpace() == XAttr.NameSpace.TRUSTED) {
return true;
}
return false;
}
List<XAttr> getXAttrs(String src) throws IOException {
String srcs = normalizePath(src);
readLock();
try {
INodesInPath iip = getLastINodeInPath(srcs, true);
INode inode = resolveLastINode(src, iip);
int snapshotId = iip.getPathSnapshotId();
return unprotectedGetXAttrs(inode, snapshotId);
} finally {
readUnlock();
}
}
List<XAttr> getXAttrs(INode inode, int snapshotId) throws IOException {
readLock();
try {
return unprotectedGetXAttrs(inode, snapshotId);
} finally {
readUnlock();
}
}
private List<XAttr> unprotectedGetXAttrs(INode inode, int snapshotId)
throws IOException {
return XAttrStorage.readINodeXAttrs(inode, snapshotId);
}
private XAttr unprotectedGetXAttrByName(INode inode, int snapshotId,
String xAttrName)
throws IOException {
List<XAttr> xAttrs = XAttrStorage.readINodeXAttrs(inode, snapshotId);
if (xAttrs == null) {
return null;
}
for (XAttr x : xAttrs) {
if (XAttrHelper.getPrefixName(x)
.equals(xAttrName)) {
return x;
}
}
return null;
}
static INode resolveLastINode(String src, INodesInPath iip)
throws FileNotFoundException {
INode[] inodes = iip.getINodes();

19
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
View File

@ -537,7 +537,8 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
}
case OP_SET_GENSTAMP_V1: {
SetGenstampV1Op setGenstampV1Op = (SetGenstampV1Op)op;
fsNamesys.getBlockIdManager().setGenerationStampV1(setGenstampV1Op.genStampV1);
fsNamesys.getBlockIdManager().setGenerationStampV1(
setGenstampV1Op.genStampV1);
break;
}
case OP_SET_PERMISSIONS: {
@ -730,12 +731,14 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
}
case OP_SET_GENSTAMP_V2: {
SetGenstampV2Op setGenstampV2Op = (SetGenstampV2Op) op;
fsNamesys.getBlockIdManager().setGenerationStampV2(setGenstampV2Op.genStampV2);
fsNamesys.getBlockIdManager().setGenerationStampV2(
setGenstampV2Op.genStampV2);
break;
}
case OP_ALLOCATE_BLOCK_ID: {
AllocateBlockIdOp allocateBlockIdOp = (AllocateBlockIdOp) op;
fsNamesys.getBlockIdManager().setLastAllocatedBlockId(allocateBlockIdOp.blockId);
fsNamesys.getBlockIdManager().setLastAllocatedBlockId(
allocateBlockIdOp.blockId);
break;
}
case OP_ROLLING_UPGRADE_START: {
@ -828,8 +831,10 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
}
case OP_SET_XATTR: {
SetXAttrOp setXAttrOp = (SetXAttrOp) op;
fsDir.unprotectedSetXAttrs(setXAttrOp.src, setXAttrOp.xAttrs,
EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE));
FSDirXAttrOp.unprotectedSetXAttrs(fsDir, setXAttrOp.src,
setXAttrOp.xAttrs,
EnumSet.of(XAttrSetFlag.CREATE,
XAttrSetFlag.REPLACE));
if (toAddRetryCache) {
fsNamesys.addCacheEntry(setXAttrOp.rpcClientId, setXAttrOp.rpcCallId);
}
@ -837,8 +842,8 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
}
case OP_REMOVE_XATTR: {
RemoveXAttrOp removeXAttrOp = (RemoveXAttrOp) op;
fsDir.unprotectedRemoveXAttrs(removeXAttrOp.src,
removeXAttrOp.xAttrs);
FSDirXAttrOp.unprotectedRemoveXAttrs(fsDir, removeXAttrOp.src,
removeXAttrOp.xAttrs);
if (toAddRetryCache) {
fsNamesys.addCacheEntry(removeXAttrOp.rpcClientId,
removeXAttrOp.rpcCallId);

227
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
View File

@ -532,9 +532,6 @@ private void logAuditEvent(boolean succeeded,
private final RetryCache retryCache;
private final boolean xattrsEnabled;
private final int xattrMaxSize;
private KeyProviderCryptoExtension provider = null;
private volatile boolean imageLoaded = false;
@ -849,19 +846,6 @@ static FSNamesystem loadFromDisk(Configuration conf) throws IOException {
this.isDefaultAuditLogger = auditLoggers.size() == 1 &&
auditLoggers.get(0) instanceof DefaultAuditLogger;
this.retryCache = ignoreRetryCache ? null : initRetryCache(conf);
this.xattrsEnabled = conf.getBoolean(
DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY,
DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_DEFAULT);
LOG.info("XAttrs enabled? " + xattrsEnabled);
this.xattrMaxSize = conf.getInt(
DFSConfigKeys.DFS_NAMENODE_MAX_XATTR_SIZE_KEY,
DFSConfigKeys.DFS_NAMENODE_MAX_XATTR_SIZE_DEFAULT);
Preconditions.checkArgument(xattrMaxSize >= 0,
"Cannot set a negative value for the maximum size of an xattr (%s).",
DFSConfigKeys.DFS_NAMENODE_MAX_XATTR_SIZE_KEY);
final String unlimited = xattrMaxSize == 0 ? " (unlimited)" : "";
LOG.info("Maximum size of an xattr: " + xattrMaxSize + unlimited);
} catch(IOException e) {
LOG.error(getClass().getSimpleName() + " initialization failed.", e);
close();
@ -5827,7 +5811,7 @@ private void checkUnreadableBySuperuser(FSPermissionChecker pc,
INode inode, int snapshotId)
throws IOException {
if (pc.isSuperUser()) {
for (XAttr xattr : dir.getXAttrs(inode, snapshotId)) {
for (XAttr xattr : FSDirXAttrOp.getXAttrs(dir, inode, snapshotId)) {
if (XAttrHelper.getPrefixName(xattr).
equals(SECURITY_XATTR_UNREADABLE_BY_SUPERUSER)) {
throw new AccessControlException("Access is denied for " +
@ -7967,136 +7951,35 @@ BatchedListEntries<EncryptionZone> listEncryptionZones(long prevId)
}
}
/**
* Set xattr for a file or directory.
*
* @param src
* - path on which it sets the xattr
* @param xAttr
* - xAttr details to set
* @param flag
* - xAttrs flags
* @throws AccessControlException
* @throws SafeModeException
* @throws UnresolvedLinkException
* @throws IOException
*/
void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag,
boolean logRetryCache)
throws AccessControlException, SafeModeException,
UnresolvedLinkException, IOException {
try {
setXAttrInt(src, xAttr, flag, logRetryCache);
} catch (AccessControlException e) {
logAuditEvent(false, "setXAttr", src);
throw e;
}
}
private void setXAttrInt(final String srcArg, XAttr xAttr,
EnumSet<XAttrSetFlag> flag, boolean logRetryCache) throws IOException {
String src = srcArg;
checkXAttrsConfigFlag();
checkXAttrSize(xAttr);
HdfsFileStatus resultingStat = null;
FSPermissionChecker pc = getPermissionChecker();
XAttrPermissionFilter.checkPermissionForApi(pc, xAttr,
FSDirectory.isReservedRawName(src));
throws IOException {
checkOperation(OperationCategory.WRITE);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
HdfsFileStatus auditStat = null;
writeLock();
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot set XAttr on " + src);
src = dir.resolvePath(pc, src, pathComponents);
final INodesInPath iip = dir.getINodesInPath4Write(src);
checkXAttrChangeAccess(iip, xAttr, pc);
List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
xAttrs.add(xAttr);
dir.setXAttrs(src, xAttrs, flag);
getEditLog().logSetXAttrs(src, xAttrs, logRetryCache);
resultingStat = getAuditFileInfo(src, false);
auditStat = FSDirXAttrOp.setXAttr(dir, src, xAttr, flag, logRetryCache);
} catch (AccessControlException e) {
logAuditEvent(false, "setXAttr", src);
throw e;
} finally {
writeUnlock();
}
getEditLog().logSync();
logAuditEvent(true, "setXAttr", srcArg, null, resultingStat);
logAuditEvent(true, "setXAttr", src, null, auditStat);
}
/**
* Verifies that the combined size of the name and value of an xattr is within
* the configured limit. Setting a limit of zero disables this check.
*/
private void checkXAttrSize(XAttr xAttr) {
if (xattrMaxSize == 0) {
return;
}
int size = xAttr.getName().getBytes(Charsets.UTF_8).length;
if (xAttr.getValue() != null) {
size += xAttr.getValue().length;
}
if (size > xattrMaxSize) {
throw new HadoopIllegalArgumentException(
"The XAttr is too big. The maximum combined size of the"
+ " name and value is " + xattrMaxSize
+ ", but the total size is " + size);
}
}
List<XAttr> getXAttrs(final String srcArg, List<XAttr> xAttrs)
List<XAttr> getXAttrs(final String src, List<XAttr> xAttrs)
throws IOException {
String src = srcArg;
checkXAttrsConfigFlag();
FSPermissionChecker pc = getPermissionChecker();
final boolean isRawPath = FSDirectory.isReservedRawName(src);
boolean getAll = xAttrs == null || xAttrs.isEmpty();
if (!getAll) {
try {
XAttrPermissionFilter.checkPermissionForApi(pc, xAttrs, isRawPath);
} catch (AccessControlException e) {
logAuditEvent(false, "getXAttrs", srcArg);
throw e;
}
}
checkOperation(OperationCategory.READ);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
readLock();
try {
checkOperation(OperationCategory.READ);
src = dir.resolvePath(pc, src, pathComponents);
final INodesInPath iip = dir.getINodesInPath(src, true);
if (isPermissionEnabled) {
dir.checkPathAccess(pc, iip, FsAction.READ);
}
List<XAttr> all = dir.getXAttrs(src);
List<XAttr> filteredAll = XAttrPermissionFilter.
filterXAttrsForApi(pc, all, isRawPath);
if (getAll) {
return filteredAll;
} else {
if (filteredAll == null || filteredAll.isEmpty()) {
return null;
}
List<XAttr> toGet = Lists.newArrayListWithCapacity(xAttrs.size());
for (XAttr xAttr : xAttrs) {
boolean foundIt = false;
for (XAttr a : filteredAll) {
if (xAttr.getNameSpace() == a.getNameSpace()
&& xAttr.getName().equals(a.getName())) {
toGet.add(a);
foundIt = true;
break;
}
}
if (!foundIt) {
throw new IOException(
"At least one of the attributes provided was not found.");
}
}
return toGet;
}
return FSDirXAttrOp.getXAttrs(dir, src, xAttrs);
} catch (AccessControlException e) {
logAuditEvent(false, "getXAttrs", srcArg);
logAuditEvent(false, "getXAttrs", src);
throw e;
} finally {
readUnlock();
@ -8104,23 +7987,11 @@ List<XAttr> getXAttrs(final String srcArg, List<XAttr> xAttrs)
}
List<XAttr> listXAttrs(String src) throws IOException {
checkXAttrsConfigFlag();
final FSPermissionChecker pc = getPermissionChecker();
final boolean isRawPath = FSDirectory.isReservedRawName(src);
checkOperation(OperationCategory.READ);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
readLock();
try {
checkOperation(OperationCategory.READ);
src = dir.resolvePath(pc, src, pathComponents);
final INodesInPath iip = dir.getINodesInPath(src, true);
if (isPermissionEnabled) {
/* To access xattr names, you need EXECUTE in the owning directory. */
dir.checkParentAccess(pc, iip, FsAction.EXECUTE);
}
final List<XAttr> all = dir.getXAttrs(src);
return XAttrPermissionFilter.
filterXAttrsForApi(pc, all, isRawPath);
return FSDirXAttrOp.listXAttrs(dir, src);
} catch (AccessControlException e) {
logAuditEvent(false, "listXAttrs", src);
throw e;
@ -8129,77 +8000,23 @@ List<XAttr> listXAttrs(String src) throws IOException {
}
}
/**
* Remove an xattr for a file or directory.
*
* @param src
* - path to remove the xattr from
* @param xAttr
* - xAttr to remove
* @throws AccessControlException
* @throws SafeModeException
* @throws UnresolvedLinkException
* @throws IOException
*/
void removeXAttr(String src, XAttr xAttr, boolean logRetryCache)
throws IOException {
try {
removeXAttrInt(src, xAttr, logRetryCache);
} catch (AccessControlException e) {
logAuditEvent(false, "removeXAttr", src);
throw e;
}
}
void removeXAttrInt(final String srcArg, XAttr xAttr, boolean logRetryCache)
throws IOException {
String src = srcArg;
checkXAttrsConfigFlag();
HdfsFileStatus resultingStat = null;
FSPermissionChecker pc = getPermissionChecker();
XAttrPermissionFilter.checkPermissionForApi(pc, xAttr,
FSDirectory.isReservedRawName(src));
checkOperation(OperationCategory.WRITE);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
HdfsFileStatus auditStat = null;
writeLock();
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot remove XAttr entry on " + src);
src = dir.resolvePath(pc, src, pathComponents);
final INodesInPath iip = dir.getINodesInPath4Write(src);
checkXAttrChangeAccess(iip, xAttr, pc);
List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
xAttrs.add(xAttr);
List<XAttr> removedXAttrs = dir.removeXAttrs(src, xAttrs);
if (removedXAttrs != null && !removedXAttrs.isEmpty()) {
getEditLog().logRemoveXAttrs(src, removedXAttrs, logRetryCache);
} else {
throw new IOException(
"No matching attributes found for remove operation");
}
resultingStat = getAuditFileInfo(src, false);
auditStat = FSDirXAttrOp.removeXAttr(dir, src, xAttr, logRetryCache);
} catch (AccessControlException e) {
logAuditEvent(false, "removeXAttr", src);
throw e;
} finally {
writeUnlock();
}
getEditLog().logSync();
logAuditEvent(true, "removeXAttr", srcArg, null, resultingStat);
}
private void checkXAttrChangeAccess(INodesInPath iip, XAttr xAttr,
FSPermissionChecker pc) throws AccessControlException {
if (isPermissionEnabled && xAttr.getNameSpace() == XAttr.NameSpace.USER) {
final INode inode = iip.getLastINode();
if (inode != null &&
inode.isDirectory() &&
inode.getFsPermission().getStickyBit()) {
if (!pc.isSuperUser()) {
dir.checkOwner(pc, iip);
}
} else {
dir.checkPathAccess(pc, iip, FsAction.WRITE);
}
}
logAuditEvent(true, "removeXAttr", src, null, auditStat);
}
void checkAccess(String src, FsAction mode) throws IOException {
@ -8311,13 +8128,5 @@ private static void enableAsyncAuditLog() {
}
}
private void checkXAttrsConfigFlag() throws IOException {
if (!xattrsEnabled) {
throw new IOException(String.format(
"The XAttr operation has been rejected. "
+ "Support for XAttrs has been disabled by setting %s to false.",
DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY));
}
}
}

47
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java
View File

@ -40,7 +40,6 @@
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@ -201,8 +200,9 @@ public void testINodeXAttrsLimit() throws Exception {
List<XAttr> newXAttrs = Lists.newArrayListWithCapacity(2);
newXAttrs.add(newSystemXAttr);
newXAttrs.add(newRawXAttr);
List<XAttr> xAttrs = fsdir.setINodeXAttrs(existingXAttrs, newXAttrs,
EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE));
List<XAttr> xAttrs = FSDirXAttrOp.setINodeXAttrs(fsdir, existingXAttrs,
newXAttrs, EnumSet.of(
XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE));
assertEquals(xAttrs.size(), 4);
// Adding a trusted namespace xAttr, is affected by inode xAttrs limit.
@ -211,8 +211,9 @@ public void testINodeXAttrsLimit() throws Exception {
setValue(new byte[]{0x34, 0x34, 0x34}).build();
newXAttrs.set(0, newXAttr1);
try {
fsdir.setINodeXAttrs(existingXAttrs, newXAttrs,
EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE));
FSDirXAttrOp.setINodeXAttrs(fsdir, existingXAttrs, newXAttrs,
EnumSet.of(XAttrSetFlag.CREATE,
XAttrSetFlag.REPLACE));
fail("Setting user visible xattr on inode should fail if " +
"reaching limit.");
} catch (IOException e) {
@ -275,8 +276,9 @@ public void testXAttrMultiSetRemove() throws Exception {
for (int i = 0; i < toAdd.size(); i++) {
LOG.info("Will add XAttr " + toAdd.get(i));
}
List<XAttr> newXAttrs = fsdir.setINodeXAttrs(existingXAttrs, toAdd,
EnumSet.of(XAttrSetFlag.CREATE));
List<XAttr> newXAttrs = FSDirXAttrOp.setINodeXAttrs(fsdir, existingXAttrs,
toAdd, EnumSet.of(
XAttrSetFlag.CREATE));
verifyXAttrsPresent(newXAttrs, numExpectedXAttrs);
existingXAttrs = newXAttrs;
}
@ -296,8 +298,9 @@ public void testXAttrMultiSetRemove() throws Exception {
final int expectedNumToRemove = toRemove.size();
LOG.info("Attempting to remove " + expectedNumToRemove + " XAttrs");
List<XAttr> removedXAttrs = Lists.newArrayList();
List<XAttr> newXAttrs = fsdir.filterINodeXAttrs(existingXAttrs,
toRemove, removedXAttrs);
List<XAttr> newXAttrs = FSDirXAttrOp.filterINodeXAttrs(existingXAttrs,
toRemove,
removedXAttrs);
assertEquals("Unexpected number of removed XAttrs",
expectedNumToRemove, removedXAttrs.size());
verifyXAttrsPresent(newXAttrs, numExpectedXAttrs);
@ -316,8 +319,8 @@ public void testXAttrMultiAddRemoveErrors() throws Exception {
toAdd.add(generatedXAttrs.get(2));
toAdd.add(generatedXAttrs.get(0));
try {
fsdir.setINodeXAttrs(existingXAttrs, toAdd, EnumSet.of(XAttrSetFlag
.CREATE));
FSDirXAttrOp.setINodeXAttrs(fsdir, existingXAttrs, toAdd,
EnumSet.of(XAttrSetFlag.CREATE));
fail("Specified the same xattr to be set twice");
} catch (IOException e) {
GenericTestUtils.assertExceptionContains("Cannot specify the same " +
@ -328,15 +331,15 @@ public void testXAttrMultiAddRemoveErrors() throws Exception {
toAdd.remove(generatedXAttrs.get(0));
existingXAttrs.add(generatedXAttrs.get(0));
try {
fsdir.setINodeXAttrs(existingXAttrs, toAdd, EnumSet.of(XAttrSetFlag
.CREATE));
FSDirXAttrOp.setINodeXAttrs(fsdir, existingXAttrs, toAdd,
EnumSet.of(XAttrSetFlag.CREATE));
fail("Set XAttr that is already set without REPLACE flag");
} catch (IOException e) {
GenericTestUtils.assertExceptionContains("already exists", e);
}
try {
fsdir.setINodeXAttrs(existingXAttrs, toAdd, EnumSet.of(XAttrSetFlag
.REPLACE));
FSDirXAttrOp.setINodeXAttrs(fsdir, existingXAttrs, toAdd,
EnumSet.of(XAttrSetFlag.REPLACE));
fail("Set XAttr that does not exist without the CREATE flag");
} catch (IOException e) {
GenericTestUtils.assertExceptionContains("does not exist", e);
@ -344,8 +347,9 @@ public void testXAttrMultiAddRemoveErrors() throws Exception {
// Sanity test for CREATE
toAdd.remove(generatedXAttrs.get(0));
List<XAttr> newXAttrs = fsdir.setINodeXAttrs(existingXAttrs, toAdd,
EnumSet.of(XAttrSetFlag.CREATE));
List<XAttr> newXAttrs = FSDirXAttrOp.setINodeXAttrs(fsdir, existingXAttrs,
toAdd, EnumSet.of(
XAttrSetFlag.CREATE));
assertEquals("Unexpected toAdd size", 2, toAdd.size());
for (XAttr x : toAdd) {
assertTrue("Did not find added XAttr " + x, newXAttrs.contains(x));
@ -362,8 +366,8 @@ public void testXAttrMultiAddRemoveErrors() throws Exception {
.build();
toAdd.add(xAttr);
}
newXAttrs = fsdir.setINodeXAttrs(existingXAttrs, toAdd,
EnumSet.of(XAttrSetFlag.REPLACE));
newXAttrs = FSDirXAttrOp.setINodeXAttrs(fsdir, existingXAttrs, toAdd,
EnumSet.of(XAttrSetFlag.REPLACE));
assertEquals("Unexpected number of new XAttrs", 3, newXAttrs.size());
for (int i=0; i<3; i++) {
assertArrayEquals("Unexpected XAttr value",
@ -376,8 +380,9 @@ public void testXAttrMultiAddRemoveErrors() throws Exception {
for (int i=0; i<4; i++) {
toAdd.add(generatedXAttrs.get(i));
}
newXAttrs = fsdir.setINodeXAttrs(existingXAttrs, toAdd,
EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE));
newXAttrs = FSDirXAttrOp.setINodeXAttrs(fsdir, existingXAttrs, toAdd,
EnumSet.of(XAttrSetFlag.CREATE,
XAttrSetFlag.REPLACE));
verifyXAttrsPresent(newXAttrs, 4);
}
}