big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-10832. Propagate ACL bit and isEncrypted bit in HttpFS FileStatus permissions.

Browse Source
This commit is contained in:
Andrew Wang 2016-09-09 12:12:29 -07:00
parent 9f192cc5ac
commit cba973f036
4 changed files with 229 additions and 230 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
View File

@ -38,6 +38,7 @@
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.FsPermissionExtension;
import org.apache.hadoop.lib.wsrs.EnumSetParam;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
@ -181,6 +182,8 @@ public static FILE_TYPE getType(FileStatus fileStatus) {
public static final String ACL_ENTRIES_JSON = "entries";
public static final String ACL_BIT_JSON = "aclBit";
public static final String ENC_BIT_JSON = "encBit";
public static final int HTTP_TEMPORARY_REDIRECT = 307;
private static final String HTTP_GET = "GET";
@ -955,6 +958,21 @@ public AclStatus getAclStatus(Path path) throws IOException {
return createAclStatus(json);
}
/** Convert a string to a FsPermission object. */
static FsPermission toFsPermission(JSONObject json) {
final String s = (String) json.get(PERMISSION_JSON);
final Boolean aclBit = (Boolean) json.get(ACL_BIT_JSON);
final Boolean encBit = (Boolean) json.get(ENC_BIT_JSON);
FsPermission perm = new FsPermission(Short.parseShort(s, 8));
final boolean aBit = (aclBit != null) ? aclBit : false;
final boolean eBit = (encBit != null) ? encBit : false;
if (aBit || eBit) {
return new FsPermissionExtension(perm, aBit, eBit);
} else {
return perm;
}
}
private FileStatus createFileStatus(Path parent, JSONObject json) {
String pathSuffix = (String) json.get(PATH_SUFFIX_JSON);
Path path = (pathSuffix.equals("")) ? parent : new Path(parent, pathSuffix);
@ -962,8 +980,7 @@ private FileStatus createFileStatus(Path parent, JSONObject json) {
long len = (Long) json.get(LENGTH_JSON);
String owner = (String) json.get(OWNER_JSON);
String group = (String) json.get(GROUP_JSON);
FsPermission permission =
new FsPermission(Short.parseShort((String) json.get(PERMISSION_JSON), 8));
final FsPermission permission = toFsPermission(json);
long aTime = (Long) json.get(ACCESS_TIME_JSON);
long mTime = (Long) json.get(MODIFICATION_TIME_JSON);
long blockSize = (Long) json.get(BLOCK_SIZE_JSON);

188
hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
View File

@ -31,7 +31,6 @@
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.protocol.AclException;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.lib.service.FileSystemAccess;
import org.apache.hadoop.util.StringUtils;
@ -54,148 +53,59 @@
public class FSOperations {
/**
* This class is used to group a FileStatus and an AclStatus together.
* It's needed for the GETFILESTATUS and LISTSTATUS calls, which take
* most info from the FileStatus and a wee bit from the AclStatus.
* @param fileStatus a FileStatus object
* @return JSON map suitable for wire transport
*/
private static class StatusPair {
private FileStatus fileStatus;
private AclStatus aclStatus;
/**
* Simple constructor
* @param fileStatus Existing FileStatus object
* @param aclStatus Existing AclStatus object
*/
public StatusPair(FileStatus fileStatus, AclStatus aclStatus) {
this.fileStatus = fileStatus;
this.aclStatus = aclStatus;
}
/**
* Create one StatusPair by performing the underlying calls to
* fs.getFileStatus and fs.getAclStatus
* @param fs The FileSystem where 'path' lives
* @param path The file/directory to query
* @throws IOException
*/
public StatusPair(FileSystem fs, Path path) throws IOException {
fileStatus = fs.getFileStatus(path);
aclStatus = null;
try {
aclStatus = fs.getAclStatus(path);
} catch (AclException e) {
/*
* The cause is almost certainly an "ACLS aren't enabled"
* exception, so leave aclStatus at null and carry on.
*/
} catch (UnsupportedOperationException e) {
/* Ditto above - this is the case for a local file system */
}
}
/**
* Return a Map suitable for conversion into JSON format
* @return The JSONish Map
*/
public Map<String,Object> toJson() {
Map<String,Object> json = new LinkedHashMap<String,Object>();
json.put(HttpFSFileSystem.FILE_STATUS_JSON, toJsonInner(true));
return json;
}
/**
* Return in inner part of the JSON for the status - used by both the
* GETFILESTATUS and LISTSTATUS calls.
* @param emptyPathSuffix Whether or not to include PATH_SUFFIX_JSON
* @return The JSONish Map
*/
public Map<String,Object> toJsonInner(boolean emptyPathSuffix) {
Map<String,Object> json = new LinkedHashMap<String,Object>();
json.put(HttpFSFileSystem.PATH_SUFFIX_JSON,
(emptyPathSuffix) ? "" : fileStatus.getPath().getName());
json.put(HttpFSFileSystem.TYPE_JSON,
HttpFSFileSystem.FILE_TYPE.getType(fileStatus).toString());
json.put(HttpFSFileSystem.LENGTH_JSON, fileStatus.getLen());
json.put(HttpFSFileSystem.OWNER_JSON, fileStatus.getOwner());
json.put(HttpFSFileSystem.GROUP_JSON, fileStatus.getGroup());
json.put(HttpFSFileSystem.PERMISSION_JSON,
HttpFSFileSystem.permissionToString(fileStatus.getPermission()));
json.put(HttpFSFileSystem.ACCESS_TIME_JSON, fileStatus.getAccessTime());
json.put(HttpFSFileSystem.MODIFICATION_TIME_JSON,
fileStatus.getModificationTime());
json.put(HttpFSFileSystem.BLOCK_SIZE_JSON, fileStatus.getBlockSize());
json.put(HttpFSFileSystem.REPLICATION_JSON, fileStatus.getReplication());
if ( (aclStatus != null) && !(aclStatus.getEntries().isEmpty()) ) {
json.put(HttpFSFileSystem.ACL_BIT_JSON,true);
}
return json;
}
private static Map<String, Object> toJson(FileStatus fileStatus) {
Map<String, Object> json = new LinkedHashMap<>();
json.put(HttpFSFileSystem.FILE_STATUS_JSON, toJsonInner(fileStatus, true));
return json;
}
/**
* Simple class used to contain and operate upon a list of StatusPair
* objects. Used by LISTSTATUS.
* @param fileStatuses list of FileStatus objects
* @return JSON map suitable for wire transport
*/
private static class StatusPairs {
private StatusPair[] statusPairs;
/**
* Construct a list of StatusPair objects
* @param fs The FileSystem where 'path' lives
* @param path The directory to query
* @param filter A possible filter for entries in the directory
* @throws IOException
*/
public StatusPairs(FileSystem fs, Path path, PathFilter filter)
throws IOException {
/* Grab all the file statuses at once in an array */
FileStatus[] fileStatuses = fs.listStatus(path, filter);
/* We'll have an array of StatusPairs of the same length */
AclStatus aclStatus = null;
statusPairs = new StatusPair[fileStatuses.length];
/*
* For each FileStatus, attempt to acquire an AclStatus. If the
* getAclStatus throws an exception, we assume that ACLs are turned
* off entirely and abandon the attempt.
*/
boolean useAcls = true; // Assume ACLs work until proven otherwise
for (int i = 0; i < fileStatuses.length; i++) {
if (useAcls) {
try {
aclStatus = fs.getAclStatus(fileStatuses[i].getPath());
} catch (AclException e) {
/* Almost certainly due to an "ACLs not enabled" exception */
aclStatus = null;
useAcls = false;
} catch (UnsupportedOperationException e) {
/* Ditto above - this is the case for a local file system */
aclStatus = null;
useAcls = false;
}
}
statusPairs[i] = new StatusPair(fileStatuses[i], aclStatus);
}
@SuppressWarnings({"unchecked"})
private static Map<String, Object> toJson(FileStatus[] fileStatuses) {
Map<String, Object> json = new LinkedHashMap<>();
Map<String, Object> inner = new LinkedHashMap<>();
JSONArray statuses = new JSONArray();
for (FileStatus f : fileStatuses) {
statuses.add(toJsonInner(f, false));
}
inner.put(HttpFSFileSystem.FILE_STATUS_JSON, statuses);
json.put(HttpFSFileSystem.FILE_STATUSES_JSON, inner);
return json;
}
/**
* Return a Map suitable for conversion into JSON.
* @return A JSONish Map
*/
@SuppressWarnings({"unchecked"})
public Map<String,Object> toJson() {
Map<String,Object> json = new LinkedHashMap<String,Object>();
Map<String,Object> inner = new LinkedHashMap<String,Object>();
JSONArray statuses = new JSONArray();
for (StatusPair s : statusPairs) {
statuses.add(s.toJsonInner(false));
}
inner.put(HttpFSFileSystem.FILE_STATUS_JSON, statuses);
json.put(HttpFSFileSystem.FILE_STATUSES_JSON, inner);
return json;
/**
* Not meant to be called directly except by the other toJson functions.
*/
private static Map<String, Object> toJsonInner(FileStatus fileStatus,
boolean emptyPathSuffix) {
Map<String, Object> json = new LinkedHashMap<String, Object>();
json.put(HttpFSFileSystem.PATH_SUFFIX_JSON,
(emptyPathSuffix) ? "" : fileStatus.getPath().getName());
json.put(HttpFSFileSystem.TYPE_JSON,
HttpFSFileSystem.FILE_TYPE.getType(fileStatus).toString());
json.put(HttpFSFileSystem.LENGTH_JSON, fileStatus.getLen());
json.put(HttpFSFileSystem.OWNER_JSON, fileStatus.getOwner());
json.put(HttpFSFileSystem.GROUP_JSON, fileStatus.getGroup());
json.put(HttpFSFileSystem.PERMISSION_JSON,
HttpFSFileSystem.permissionToString(fileStatus.getPermission()));
json.put(HttpFSFileSystem.ACCESS_TIME_JSON, fileStatus.getAccessTime());
json.put(HttpFSFileSystem.MODIFICATION_TIME_JSON,
fileStatus.getModificationTime());
json.put(HttpFSFileSystem.BLOCK_SIZE_JSON, fileStatus.getBlockSize());
json.put(HttpFSFileSystem.REPLICATION_JSON, fileStatus.getReplication());
if (fileStatus.getPermission().getAclBit()) {
json.put(HttpFSFileSystem.ACL_BIT_JSON, true);
}
if (fileStatus.getPermission().getEncryptedBit()) {
json.put(HttpFSFileSystem.ENC_BIT_JSON, true);
}
return json;
}
/** Converts an <code>AclStatus</code> object into a JSON object.
@ -637,8 +547,8 @@ public FSFileStatus(String path) {
*/
@Override
public Map execute(FileSystem fs) throws IOException {
StatusPair sp = new StatusPair(fs, path);
return sp.toJson();
FileStatus status = fs.getFileStatus(path);
return toJson(status);
}
}
@ -703,8 +613,8 @@ public FSListStatus(String path, String filter) throws IOException {
*/
@Override
public Map execute(FileSystem fs) throws IOException {
StatusPairs sp = new StatusPairs(fs, path, filter);
return sp.toJson();
FileStatus[] fileStatuses = fs.listStatus(path, filter);
return toJson(fileStatuses);
}
@Override

227
hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
View File

@ -40,6 +40,7 @@
import org.apache.hadoop.test.TestDir;
import org.apache.hadoop.test.TestDirHelper;
import org.apache.hadoop.test.TestHdfs;
import org.apache.hadoop.test.TestHdfsHelper;
import org.apache.hadoop.test.TestJetty;
import org.apache.hadoop.test.TestJettyHelper;
import org.junit.Assert;
@ -66,6 +67,11 @@
import java.util.List;
import java.util.Map;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@RunWith(value = Parameterized.class)
public abstract class BaseTestHttpFSWith extends HFSTestCase {
@ -81,9 +87,9 @@ protected boolean isLocalFS() {
private void createHttpFSServer() throws Exception {
File homeDir = TestDirHelper.getTestDir();
Assert.assertTrue(new File(homeDir, "conf").mkdir());
Assert.assertTrue(new File(homeDir, "log").mkdir());
Assert.assertTrue(new File(homeDir, "temp").mkdir());
assertTrue(new File(homeDir, "conf").mkdir());
assertTrue(new File(homeDir, "log").mkdir());
assertTrue(new File(homeDir, "temp").mkdir());
HttpFSServerWebApp.setHomeDirForCurrentThread(homeDir.getAbsolutePath());
File secretFile = new File(new File(homeDir, "conf"), "secret");
@ -143,7 +149,7 @@ protected void testGet() throws Exception {
Assert.assertNotNull(fs);
URI uri = new URI(getScheme() + "://" +
TestJettyHelper.getJettyURL().toURI().getAuthority());
Assert.assertEquals(fs.getUri(), uri);
assertEquals(fs.getUri(), uri);
fs.close();
}
@ -156,7 +162,7 @@ private void testOpen() throws Exception {
fs.close();
fs = getHttpFSFileSystem();
InputStream is = fs.open(new Path(path.toUri().getPath()));
Assert.assertEquals(is.read(), 1);
assertEquals(is.read(), 1);
is.close();
fs.close();
}
@ -173,12 +179,12 @@ private void testCreate(Path path, boolean override) throws Exception {
fs = FileSystem.get(getProxiedFSConf());
FileStatus status = fs.getFileStatus(path);
if (!isLocalFS()) {
Assert.assertEquals(status.getReplication(), 2);
Assert.assertEquals(status.getBlockSize(), 100 * 1024 * 1024);
assertEquals(status.getReplication(), 2);
assertEquals(status.getBlockSize(), 100 * 1024 * 1024);
}
Assert.assertEquals(status.getPermission(), permission);
assertEquals(status.getPermission(), permission);
InputStream is = fs.open(path);
Assert.assertEquals(is.read(), 1);
assertEquals(is.read(), 1);
is.close();
fs.close();
}
@ -216,9 +222,9 @@ private void testAppend() throws Exception {
fs.close();
fs = FileSystem.get(getProxiedFSConf());
InputStream is = fs.open(path);
Assert.assertEquals(is.read(), 1);
Assert.assertEquals(is.read(), 2);
Assert.assertEquals(is.read(), -1);
assertEquals(is.read(), 1);
assertEquals(is.read(), 2);
assertEquals(is.read(), -1);
is.close();
fs.close();
}
@ -239,10 +245,10 @@ private void testTruncate() throws Exception {
final int newLength = blockSize;
boolean isReady = fs.truncate(file, newLength);
Assert.assertTrue("Recovery is not expected.", isReady);
assertTrue("Recovery is not expected.", isReady);
FileStatus fileStatus = fs.getFileStatus(file);
Assert.assertEquals(fileStatus.getLen(), newLength);
assertEquals(fileStatus.getLen(), newLength);
AppendTestUtil.checkFullFile(fs, file, newLength, data, file.toString());
fs.close();
@ -266,9 +272,9 @@ private void testConcat() throws Exception {
fs.concat(path1, new Path[]{path2, path3});
fs.close();
fs = FileSystem.get(config);
Assert.assertTrue(fs.exists(path1));
Assert.assertFalse(fs.exists(path2));
Assert.assertFalse(fs.exists(path3));
assertTrue(fs.exists(path1));
assertFalse(fs.exists(path2));
assertFalse(fs.exists(path3));
fs.close();
}
}
@ -284,8 +290,8 @@ private void testRename() throws Exception {
fs.rename(oldPath, newPath);
fs.close();
fs = FileSystem.get(getProxiedFSConf());
Assert.assertFalse(fs.exists(oldPath));
Assert.assertTrue(fs.exists(newPath));
assertFalse(fs.exists(oldPath));
assertTrue(fs.exists(newPath));
fs.close();
}
@ -299,8 +305,8 @@ private void testDelete() throws Exception {
fs.mkdirs(foe);
FileSystem hoopFs = getHttpFSFileSystem();
Assert.assertTrue(hoopFs.delete(new Path(foo.toUri().getPath()), false));
Assert.assertFalse(fs.exists(foo));
assertTrue(hoopFs.delete(new Path(foo.toUri().getPath()), false));
assertFalse(fs.exists(foo));
try {
hoopFs.delete(new Path(bar.toUri().getPath()), false);
Assert.fail();
@ -308,13 +314,13 @@ private void testDelete() throws Exception {
} catch (Exception ex) {
Assert.fail();
}
Assert.assertTrue(fs.exists(bar));
Assert.assertTrue(hoopFs.delete(new Path(bar.toUri().getPath()), true));
Assert.assertFalse(fs.exists(bar));
assertTrue(fs.exists(bar));
assertTrue(hoopFs.delete(new Path(bar.toUri().getPath()), true));
assertFalse(fs.exists(bar));
Assert.assertTrue(fs.exists(foe));
Assert.assertTrue(hoopFs.delete(foe, true));
Assert.assertFalse(fs.exists(foe));
assertTrue(fs.exists(foe));
assertTrue(hoopFs.delete(foe, true));
assertFalse(fs.exists(foe));
hoopFs.close();
fs.close();
@ -333,19 +339,20 @@ private void testListStatus() throws Exception {
FileStatus status2 = fs.getFileStatus(new Path(path.toUri().getPath()));
fs.close();
Assert.assertEquals(status2.getPermission(), status1.getPermission());
Assert.assertEquals(status2.getPath().toUri().getPath(), status1.getPath().toUri().getPath());
Assert.assertEquals(status2.getReplication(), status1.getReplication());
Assert.assertEquals(status2.getBlockSize(), status1.getBlockSize());
Assert.assertEquals(status2.getAccessTime(), status1.getAccessTime());
Assert.assertEquals(status2.getModificationTime(), status1.getModificationTime());
Assert.assertEquals(status2.getOwner(), status1.getOwner());
Assert.assertEquals(status2.getGroup(), status1.getGroup());
Assert.assertEquals(status2.getLen(), status1.getLen());
assertEquals(status2.getPermission(), status1.getPermission());
assertEquals(status2.getPath().toUri().getPath(),
status1.getPath().toUri().getPath());
assertEquals(status2.getReplication(), status1.getReplication());
assertEquals(status2.getBlockSize(), status1.getBlockSize());
assertEquals(status2.getAccessTime(), status1.getAccessTime());
assertEquals(status2.getModificationTime(), status1.getModificationTime());
assertEquals(status2.getOwner(), status1.getOwner());
assertEquals(status2.getGroup(), status1.getGroup());
assertEquals(status2.getLen(), status1.getLen());
FileStatus[] stati = fs.listStatus(path.getParent());
Assert.assertEquals(stati.length, 1);
Assert.assertEquals(stati[0].getPath().getName(), path.getName());
assertEquals(stati.length, 1);
assertEquals(stati[0].getPath().getName(), path.getName());
}
private void testWorkingdirectory() throws Exception {
@ -359,14 +366,15 @@ private void testWorkingdirectory() throws Exception {
}
Path httpFSWorkingDir = fs.getWorkingDirectory();
fs.close();
Assert.assertEquals(httpFSWorkingDir.toUri().getPath(),
assertEquals(httpFSWorkingDir.toUri().getPath(),
workingDir.toUri().getPath());
fs = getHttpFSFileSystem();
fs.setWorkingDirectory(new Path("/tmp"));
workingDir = fs.getWorkingDirectory();
fs.close();
Assert.assertEquals(workingDir.toUri().getPath(), new Path("/tmp").toUri().getPath());
assertEquals(workingDir.toUri().getPath(),
new Path("/tmp").toUri().getPath());
}
private void testMkdirs() throws Exception {
@ -375,7 +383,7 @@ private void testMkdirs() throws Exception {
fs.mkdirs(path);
fs.close();
fs = FileSystem.get(getProxiedFSConf());
Assert.assertTrue(fs.exists(path));
assertTrue(fs.exists(path));
fs.close();
}
@ -400,8 +408,8 @@ private void testSetTimes() throws Exception {
fs.close();
long atNew = status1.getAccessTime();
long mtNew = status1.getModificationTime();
Assert.assertEquals(mtNew, mt - 10);
Assert.assertEquals(atNew, at - 20);
assertEquals(mtNew, mt - 10);
assertEquals(atNew, at - 20);
}
}
@ -419,7 +427,7 @@ protected void testSetPermission() throws Exception {
FileStatus status1 = fs.getFileStatus(path);
fs.close();
FsPermission permission2 = status1.getPermission();
Assert.assertEquals(permission2, permission1);
assertEquals(permission2, permission1);
//sticky bit
fs = getHttpFSFileSystem();
@ -431,8 +439,8 @@ protected void testSetPermission() throws Exception {
status1 = fs.getFileStatus(path);
fs.close();
permission2 = status1.getPermission();
Assert.assertTrue(permission2.getStickyBit());
Assert.assertEquals(permission2, permission1);
assertTrue(permission2.getStickyBit());
assertEquals(permission2, permission1);
}
private void testSetOwner() throws Exception {
@ -454,8 +462,8 @@ private void testSetOwner() throws Exception {
fs = FileSystem.get(getProxiedFSConf());
FileStatus status1 = fs.getFileStatus(path);
fs.close();
Assert.assertEquals(status1.getOwner(), user);
Assert.assertEquals(status1.getGroup(), group);
assertEquals(status1.getOwner(), user);
assertEquals(status1.getGroup(), group);
}
}
@ -475,7 +483,7 @@ private void testSetReplication() throws Exception {
fs = FileSystem.get(getProxiedFSConf());
FileStatus status1 = fs.getFileStatus(path);
fs.close();
Assert.assertEquals(status1.getReplication(), (short) 1);
assertEquals(status1.getReplication(), (short) 1);
}
private void testChecksum() throws Exception {
@ -491,9 +499,10 @@ private void testChecksum() throws Exception {
fs = getHttpFSFileSystem();
FileChecksum httpChecksum = fs.getFileChecksum(path);
fs.close();
Assert.assertEquals(httpChecksum.getAlgorithmName(), hdfsChecksum.getAlgorithmName());
Assert.assertEquals(httpChecksum.getLength(), hdfsChecksum.getLength());
Assert.assertArrayEquals(httpChecksum.getBytes(), hdfsChecksum.getBytes());
assertEquals(httpChecksum.getAlgorithmName(),
hdfsChecksum.getAlgorithmName());
assertEquals(httpChecksum.getLength(), hdfsChecksum.getLength());
assertArrayEquals(httpChecksum.getBytes(), hdfsChecksum.getBytes());
}
}
@ -508,12 +517,17 @@ private void testContentSummary() throws Exception {
fs = getHttpFSFileSystem();
ContentSummary httpContentSummary = fs.getContentSummary(path);
fs.close();
Assert.assertEquals(httpContentSummary.getDirectoryCount(), hdfsContentSummary.getDirectoryCount());
Assert.assertEquals(httpContentSummary.getFileCount(), hdfsContentSummary.getFileCount());
Assert.assertEquals(httpContentSummary.getLength(), hdfsContentSummary.getLength());
Assert.assertEquals(httpContentSummary.getQuota(), hdfsContentSummary.getQuota());
Assert.assertEquals(httpContentSummary.getSpaceConsumed(), hdfsContentSummary.getSpaceConsumed());
Assert.assertEquals(httpContentSummary.getSpaceQuota(), hdfsContentSummary.getSpaceQuota());
assertEquals(httpContentSummary.getDirectoryCount(),
hdfsContentSummary.getDirectoryCount());
assertEquals(httpContentSummary.getFileCount(),
hdfsContentSummary.getFileCount());
assertEquals(httpContentSummary.getLength(),
hdfsContentSummary.getLength());
assertEquals(httpContentSummary.getQuota(), hdfsContentSummary.getQuota());
assertEquals(httpContentSummary.getSpaceConsumed(),
hdfsContentSummary.getSpaceConsumed());
assertEquals(httpContentSummary.getSpaceQuota(),
hdfsContentSummary.getSpaceQuota());
}
/** Set xattr */
@ -552,11 +566,11 @@ private void testSetXAttr() throws Exception {
fs = FileSystem.get(getProxiedFSConf());
Map<String, byte[]> xAttrs = fs.getXAttrs(path);
fs.close();
Assert.assertEquals(4, xAttrs.size());
Assert.assertArrayEquals(value1, xAttrs.get(name1));
Assert.assertArrayEquals(value2, xAttrs.get(name2));
Assert.assertArrayEquals(new byte[0], xAttrs.get(name3));
Assert.assertArrayEquals(value4, xAttrs.get(name4));
assertEquals(4, xAttrs.size());
assertArrayEquals(value1, xAttrs.get(name1));
assertArrayEquals(value2, xAttrs.get(name2));
assertArrayEquals(new byte[0], xAttrs.get(name3));
assertArrayEquals(value4, xAttrs.get(name4));
}
}
@ -595,16 +609,16 @@ private void testGetXAttrs() throws Exception {
names.add(name4);
Map<String, byte[]> xAttrs = fs.getXAttrs(path, names);
fs.close();
Assert.assertEquals(4, xAttrs.size());
Assert.assertArrayEquals(value1, xAttrs.get(name1));
Assert.assertArrayEquals(value2, xAttrs.get(name2));
Assert.assertArrayEquals(new byte[0], xAttrs.get(name3));
Assert.assertArrayEquals(value4, xAttrs.get(name4));
assertEquals(4, xAttrs.size());
assertArrayEquals(value1, xAttrs.get(name1));
assertArrayEquals(value2, xAttrs.get(name2));
assertArrayEquals(new byte[0], xAttrs.get(name3));
assertArrayEquals(value4, xAttrs.get(name4));
// Get specific xattr
fs = getHttpFSFileSystem();
byte[] value = fs.getXAttr(path, name1);
Assert.assertArrayEquals(value1, value);
assertArrayEquals(value1, value);
final String name5 = "a1";
try {
value = fs.getXAttr(path, name5);
@ -618,11 +632,11 @@ private void testGetXAttrs() throws Exception {
fs = getHttpFSFileSystem();
xAttrs = fs.getXAttrs(path);
fs.close();
Assert.assertEquals(4, xAttrs.size());
Assert.assertArrayEquals(value1, xAttrs.get(name1));
Assert.assertArrayEquals(value2, xAttrs.get(name2));
Assert.assertArrayEquals(new byte[0], xAttrs.get(name3));
Assert.assertArrayEquals(value4, xAttrs.get(name4));
assertEquals(4, xAttrs.size());
assertArrayEquals(value1, xAttrs.get(name1));
assertArrayEquals(value2, xAttrs.get(name2));
assertArrayEquals(new byte[0], xAttrs.get(name3));
assertArrayEquals(value4, xAttrs.get(name4));
}
}
@ -667,8 +681,8 @@ private void testRemoveXAttr() throws Exception {
fs = FileSystem.get(getProxiedFSConf());
Map<String, byte[]> xAttrs = fs.getXAttrs(path);
fs.close();
Assert.assertEquals(1, xAttrs.size());
Assert.assertArrayEquals(value2, xAttrs.get(name2));
assertEquals(1, xAttrs.size());
assertArrayEquals(value2, xAttrs.get(name2));
}
}
@ -700,11 +714,11 @@ private void testListXAttrs() throws Exception {
fs = getHttpFSFileSystem();
List<String> names = fs.listXAttrs(path);
Assert.assertEquals(4, names.size());
Assert.assertTrue(names.contains(name1));
Assert.assertTrue(names.contains(name2));
Assert.assertTrue(names.contains(name3));
Assert.assertTrue(names.contains(name4));
assertEquals(4, names.size());
assertTrue(names.contains(name1));
assertTrue(names.contains(name2));
assertTrue(names.contains(name3));
assertTrue(names.contains(name4));
}
}
@ -715,18 +729,26 @@ private void testListXAttrs() throws Exception {
* @throws Exception
*/
private void assertSameAcls(AclStatus a, AclStatus b) throws Exception {
Assert.assertTrue(a.getOwner().equals(b.getOwner()));
Assert.assertTrue(a.getGroup().equals(b.getGroup()));
Assert.assertTrue(a.isStickyBit() == b.isStickyBit());
Assert.assertTrue(a.getEntries().size() == b.getEntries().size());
assertTrue(a.getOwner().equals(b.getOwner()));
assertTrue(a.getGroup().equals(b.getGroup()));
assertTrue(a.isStickyBit() == b.isStickyBit());
assertTrue(a.getEntries().size() == b.getEntries().size());
for (AclEntry e : a.getEntries()) {
Assert.assertTrue(b.getEntries().contains(e));
assertTrue(b.getEntries().contains(e));
}
for (AclEntry e : b.getEntries()) {
Assert.assertTrue(a.getEntries().contains(e));
assertTrue(a.getEntries().contains(e));
}
}
private static void assertSameAclBit(FileSystem expected, FileSystem actual,
Path path) throws IOException {
FileStatus expectedFileStatus = expected.getFileStatus(path);
FileStatus actualFileStatus = actual.getFileStatus(path);
assertEquals(actualFileStatus.getPermission().getAclBit(),
expectedFileStatus.getPermission().getAclBit());
}
/**
* Simple ACL tests on a file: Set an acl, add an acl, remove one acl,
* and remove all acls.
@ -755,26 +777,31 @@ private void testFileAcls() throws Exception {
AclStatus proxyAclStat = proxyFs.getAclStatus(path);
AclStatus httpfsAclStat = httpfs.getAclStatus(path);
assertSameAcls(httpfsAclStat, proxyAclStat);
assertSameAclBit(httpfs, proxyFs, path);
httpfs.setAcl(path, AclEntry.parseAclSpec(aclSet,true));
proxyAclStat = proxyFs.getAclStatus(path);
httpfsAclStat = httpfs.getAclStatus(path);
assertSameAcls(httpfsAclStat, proxyAclStat);
assertSameAclBit(httpfs, proxyFs, path);
httpfs.modifyAclEntries(path, AclEntry.parseAclSpec(aclUser2, true));
proxyAclStat = proxyFs.getAclStatus(path);
httpfsAclStat = httpfs.getAclStatus(path);
assertSameAcls(httpfsAclStat, proxyAclStat);
assertSameAclBit(httpfs, proxyFs, path);
httpfs.removeAclEntries(path, AclEntry.parseAclSpec(rmAclUser1, false));
proxyAclStat = proxyFs.getAclStatus(path);
httpfsAclStat = httpfs.getAclStatus(path);
assertSameAcls(httpfsAclStat, proxyAclStat);
assertSameAclBit(httpfs, proxyFs, path);
httpfs.removeAcl(path);
proxyAclStat = proxyFs.getAclStatus(path);
httpfsAclStat = httpfs.getAclStatus(path);
assertSameAcls(httpfsAclStat, proxyAclStat);
assertSameAclBit(httpfs, proxyFs, path);
}
/**
@ -797,25 +824,46 @@ private void testDirAcls() throws Exception {
AclStatus proxyAclStat = proxyFs.getAclStatus(dir);
AclStatus httpfsAclStat = httpfs.getAclStatus(dir);
assertSameAcls(httpfsAclStat, proxyAclStat);
assertSameAclBit(httpfs, proxyFs, dir);
/* Set a default ACL on the directory */
httpfs.setAcl(dir, (AclEntry.parseAclSpec(defUser1,true)));
proxyAclStat = proxyFs.getAclStatus(dir);
httpfsAclStat = httpfs.getAclStatus(dir);
assertSameAcls(httpfsAclStat, proxyAclStat);
assertSameAclBit(httpfs, proxyFs, dir);
/* Remove the default ACL */
httpfs.removeDefaultAcl(dir);
proxyAclStat = proxyFs.getAclStatus(dir);
httpfsAclStat = httpfs.getAclStatus(dir);
assertSameAcls(httpfsAclStat, proxyAclStat);
assertSameAclBit(httpfs, proxyFs, dir);
}
private void testEncryption() throws Exception {
if (isLocalFS()) {
return;
}
FileSystem proxyFs = FileSystem.get(getProxiedFSConf());
FileSystem httpFs = getHttpFSFileSystem();
FileStatus proxyStatus = proxyFs.getFileStatus(TestHdfsHelper
.ENCRYPTED_FILE);
assertTrue(proxyStatus.isEncrypted());
FileStatus httpStatus = httpFs.getFileStatus(TestHdfsHelper
.ENCRYPTED_FILE);
assertTrue(httpStatus.isEncrypted());
proxyStatus = proxyFs.getFileStatus(new Path("/"));
httpStatus = httpFs.getFileStatus(new Path("/"));
assertFalse(proxyStatus.isEncrypted());
assertFalse(httpStatus.isEncrypted());
}
protected enum Operation {
GET, OPEN, CREATE, APPEND, TRUNCATE, CONCAT, RENAME, DELETE, LIST_STATUS,
WORKING_DIRECTORY, MKDIRS, SET_TIMES, SET_PERMISSION, SET_OWNER,
SET_REPLICATION, CHECKSUM, CONTENT_SUMMARY, FILEACLS, DIRACLS, SET_XATTR,
GET_XATTRS, REMOVE_XATTR, LIST_XATTRS
GET_XATTRS, REMOVE_XATTR, LIST_XATTRS, ENCRYPTION
}
private void operation(Operation op) throws Exception {
@ -889,6 +937,9 @@ private void operation(Operation op) throws Exception {
case LIST_XATTRS:
testListXAttrs();
break;
case ENCRYPTION:
testEncryption();
break;
}
}

23
hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestHdfsHelper.java
View File

@ -21,10 +21,14 @@
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.JavaKeyStoreProvider;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileSystemTestHelper;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.junit.Test;
import org.junit.runners.model.FrameworkMethod;
@ -129,6 +133,9 @@ public static Configuration getHdfsConf() {
return new Configuration(conf);
}
public static final Path ENCRYPTION_ZONE = new Path("/ez");
public static final Path ENCRYPTED_FILE = new Path("/ez/encfile");
private static MiniDFSCluster MINI_DFS = null;
private static synchronized MiniDFSCluster startMiniHdfs(Configuration conf) throws Exception {
@ -148,14 +155,28 @@ private static synchronized MiniDFSCluster startMiniHdfs(Configuration conf) thr
conf.set("hadoop.security.authentication", "simple");
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY, true);
FileSystemTestHelper helper = new FileSystemTestHelper();
final String jceksPath = JavaKeyStoreProvider.SCHEME_NAME + "://file" +
new Path(helper.getTestRootDir(), "test.jks").toUri();
conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, jceksPath);
MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
builder.numDataNodes(2);
MiniDFSCluster miniHdfs = builder.build();
FileSystem fileSystem = miniHdfs.getFileSystem();
final String testkey = "testkey";
DFSTestUtil.createKey(testkey, miniHdfs, conf);
DistributedFileSystem fileSystem = miniHdfs.getFileSystem();
fileSystem.getClient().setKeyProvider(miniHdfs.getNameNode()
.getNamesystem().getProvider());
fileSystem.mkdirs(new Path("/tmp"));
fileSystem.mkdirs(new Path("/user"));
fileSystem.setPermission(new Path("/tmp"), FsPermission.valueOf("-rwxrwxrwx"));
fileSystem.setPermission(new Path("/user"), FsPermission.valueOf("-rwxrwxrwx"));
fileSystem.mkdirs(ENCRYPTION_ZONE);
fileSystem.createEncryptionZone(ENCRYPTION_ZONE, testkey);
fileSystem.create(ENCRYPTED_FILE).close();
MINI_DFS = miniHdfs;
}
return MINI_DFS;