big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-14223. Extend FileStatus#toString() to include details like Erasure Coding and Encryption. Contributed by Manoj Govindassamy.

Browse Source
This commit is contained in:
Andrew Wang 2017-03-29 14:37:21 -07:00
parent 640ba1d23f
commit 4966a6e26e
10 changed files with 86 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
View File

@ -207,6 +207,15 @@ public FsPermission getPermission() {
return permission;
}
/**
* Tell whether the underlying file or directory has ACLs set.
*
* @return true if the underlying file or directory has ACLs set.
*/
public boolean hasAcl() {
return permission.getAclBit();
}
/**
* Tell whether the underlying file or directory is encrypted or not.
*
@ -399,6 +408,9 @@ public String toString() {
if(isSymlink()) {
sb.append("; symlink=" + symlink);
}
sb.append("; hasAcl=" + hasAcl());
sb.append("; isEncrypted=" + isEncrypted());
sb.append("; isErasureCoded=" + isErasureCoded());
sb.append("}");
return sb.toString();
}

19
hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
View File

@ -86,20 +86,15 @@ Get the status of a path
stat.length = 0
stat.isdir = False
stat.symlink = FS.Symlinks[p]
if inEncryptionZone(FS, p) :
stat.isEncrypted = True
else
stat.isEncrypted = False
if isErasureCoded(FS, p) :
stat.isErasureCoded = True
else
stat.isErasureCoded = False
stat.hasAcl = hasACL(FS, p)
stat.isEncrypted = inEncryptionZone(FS, p)
stat.isErasureCoded = isErasureCoded(FS, p)
The returned `FileStatus` status of the path additionally carries details on
Encryption and Erasure Coding information. `getFileStatus(Path p).isEncrypted()`
can be queried to find if the path is Encrypted.
Likewise, `getFileStatus(Path p).isErasureCoded()` will tell if the path is
Erasure Coded or not.
ACL, encryption and erasure coding information. `getFileStatus(Path p).hasAcl()`
can be queried to find if the path has an ACL. `getFileStatus(Path p).isEncrypted()`
can be queried to find if the path is encrypted. `getFileStatus(Path p).isErasureCoded()`
will tell if the path is erasure coded or not.
### `Path getHomeDirectory()`

10
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java
View File

@ -295,11 +295,15 @@ private void validateToString(FileStatus fileStatus) throws IOException {
expected.append("permission=").append(fileStatus.getPermission()).append("; ");
if(fileStatus.isSymlink()) {
expected.append("isSymlink=").append(true).append("; ");
expected.append("symlink=").append(fileStatus.getSymlink()).append("}");
expected.append("symlink=").append(fileStatus.getSymlink()).append("; ");
} else {
expected.append("isSymlink=").append(false).append("}");
expected.append("isSymlink=").append(false).append("; ");
}
expected.append("hasAcl=").append(fileStatus.hasAcl()).append("; ");
expected.append("isEncrypted=").append(
fileStatus.isEncrypted()).append("; ");
expected.append("isErasureCoded=").append(
fileStatus.isErasureCoded()).append("}");
assertEquals(expected.toString(), fileStatus.toString());
}
}

3
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewfsFileStatus.java
View File

@ -75,6 +75,9 @@ public void testFileStatusSerialziation()
FileStatus stat = vfs.getFileStatus(path);
assertEquals(content.length, stat.getLen());
ContractTestUtils.assertNotErasureCoded(vfs, path);
assertTrue(path + " should have erasure coding unset in " +
"FileStatus#toString(): " + stat,
stat.toString().contains("isErasureCoded=false"));
// check serialization/deserialization
DataOutputBuffer dob = new DataOutputBuffer();

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatus.java
View File

@ -139,6 +139,9 @@ public void testGetFileStatusOnFile() throws Exception {
assertEquals(file1.makeQualified(fs.getUri(),
fs.getWorkingDirectory()).toString(),
status.getPath().toString());
assertTrue(file1 + " should have erasure coding unset in " +
"FileStatus#toString(): " + status,
status.toString().contains("isErasureCoded=false"));
}
/** Test the FileStatus obtained calling listStatus on a file */

7
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java
View File

@ -23,6 +23,7 @@
import java.io.IOException;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.permission.FsPermission;
@ -85,7 +86,7 @@ public void testFileStatusWithECPolicy() throws Exception {
assertNotNull(ecPolicy2);
assertTrue(ecPolicy1.equals(ecPolicy2));
// test file doesn't have an EC policy
// test file with EC policy
fs.create(file).close();
final ErasureCodingPolicy ecPolicy3 =
fs.getClient().getFileInfo(file.toUri().getPath())
@ -93,5 +94,9 @@ public void testFileStatusWithECPolicy() throws Exception {
assertNotNull(ecPolicy3);
assertTrue(ecPolicy1.equals(ecPolicy3));
ContractTestUtils.assertErasureCoded(fs, file);
FileStatus status = fs.getFileStatus(file);
assertTrue(file + " should have erasure coding set in " +
"FileStatus#toString(): " + status,
status.toString().contains("isErasureCoded=true"));
}
}

23
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java
View File

@ -50,6 +50,7 @@
import org.apache.hadoop.security.UserGroupInformation;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@ -120,10 +121,16 @@ public void testModifyAclEntries() throws IOException {
aclEntry(ACCESS, OTHER, NONE),
aclEntry(DEFAULT, USER, "foo", ALL));
fs.setAcl(path, aclSpec);
Assert.assertTrue(path + " should have ACLs in FileStatus!",
fs.getFileStatus(path).hasAcl());
aclSpec = Lists.newArrayList(
aclEntry(ACCESS, USER, "foo", READ_EXECUTE),
aclEntry(DEFAULT, USER, "foo", READ_EXECUTE));
fs.modifyAclEntries(path, aclSpec);
Assert.assertTrue(path + " should have ACLs in FileStatus!",
fs.getFileStatus(path).hasAcl());
AclStatus s = fs.getAclStatus(path);
AclEntry[] returned = s.getEntries().toArray(new AclEntry[0]);
assertArrayEquals(new AclEntry[] {
@ -561,8 +568,18 @@ public void testRemoveAcl() throws IOException {
aclEntry(ACCESS, GROUP, READ_EXECUTE),
aclEntry(ACCESS, OTHER, NONE),
aclEntry(DEFAULT, USER, "foo", ALL));
fs.setAcl(path, aclSpec);
Assert.assertTrue(path + " should have ACLs in FileStatus!",
fs.getFileStatus(path).hasAcl());
Assert.assertTrue(path + " should have ACLs in FileStatus#toString()!",
fs.getFileStatus(path).toString().contains("hasAcl=true"));
fs.removeAcl(path);
Assert.assertFalse(path + " should not have ACLs in FileStatus!",
fs.getFileStatus(path).hasAcl());
Assert.assertTrue(path + " should not have ACLs in FileStatus#toString()!",
fs.getFileStatus(path).toString().contains("hasAcl=false"));
AclStatus s = fs.getAclStatus(path);
AclEntry[] returned = s.getEntries().toArray(new AclEntry[0]);
assertArrayEquals(new AclEntry[] { }, returned);
@ -968,8 +985,14 @@ public void testDefaultAclNewDir() throws Exception {
List<AclEntry> aclSpec = Lists.newArrayList(
aclEntry(DEFAULT, USER, "foo", ALL));
fs.setAcl(path, aclSpec);
Assert.assertTrue(path + " should have ACLs in FileStatus!",
fs.getFileStatus(path).hasAcl());
Path dirPath = new Path(path, "dir1");
fs.mkdirs(dirPath);
Assert.assertTrue(dirPath + " should have ACLs in FileStatus!",
fs.getFileStatus(dirPath).hasAcl());
AclStatus s = fs.getAclStatus(dirPath);
AclEntry[] returned = s.getEntries().toArray(new AclEntry[0]);
assertArrayEquals(new AclEntry[] {

12
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
View File

@ -547,24 +547,36 @@ public void testWebHdfsErasureCodingFiles() throws Exception {
Assert.assertEquals(expectedECDirStatus.isErasureCoded(),
actualECDirStatus.isErasureCoded());
ContractTestUtils.assertErasureCoded(dfs, ecDir);
assertTrue(ecDir+ " should have erasure coding set in " +
"FileStatus#toString(): " + actualECDirStatus,
actualECDirStatus.toString().contains("isErasureCoded=true"));
FileStatus expectedECFileStatus = dfs.getFileStatus(ecFile);
FileStatus actualECFileStatus = webHdfs.getFileStatus(ecFile);
Assert.assertEquals(expectedECFileStatus.isErasureCoded(),
actualECFileStatus.isErasureCoded());
ContractTestUtils.assertErasureCoded(dfs, ecFile);
assertTrue(ecFile+ " should have erasure coding set in " +
"FileStatus#toString(): " + actualECFileStatus,
actualECFileStatus.toString().contains("isErasureCoded=true"));
FileStatus expectedNormalDirStatus = dfs.getFileStatus(normalDir);
FileStatus actualNormalDirStatus = webHdfs.getFileStatus(normalDir);
Assert.assertEquals(expectedNormalDirStatus.isErasureCoded(),
actualNormalDirStatus.isErasureCoded());
ContractTestUtils.assertNotErasureCoded(dfs, normalDir);
assertTrue(normalDir + " should have erasure coding unset in " +
"FileStatus#toString(): " + actualNormalDirStatus,
actualNormalDirStatus.toString().contains("isErasureCoded=false"));
FileStatus expectedNormalFileStatus = dfs.getFileStatus(normalFile);
FileStatus actualNormalFileStatus = webHdfs.getFileStatus(normalDir);
Assert.assertEquals(expectedNormalFileStatus.isErasureCoded(),
actualNormalFileStatus.isErasureCoded());
ContractTestUtils.assertNotErasureCoded(dfs, normalFile);
assertTrue(normalFile + " should have erasure coding unset in " +
"FileStatus#toString(): " + actualNormalFileStatus,
actualNormalFileStatus.toString().contains("isErasureCoded=false"));
} finally {
if (cluster != null) {

6
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java
View File

@ -62,6 +62,9 @@ public void testFile() throws Exception {
assertEquals(meta.getContentLength(), stat.getLen());
assertEquals(meta.getLastModified().getTime(), stat.getModificationTime());
ContractTestUtils.assertNotErasureCoded(fs, path);
assertTrue(path + " should have erasure coding unset in " +
"FileStatus#toString(): " + stat,
stat.toString().contains("isErasureCoded=false"));
}
@Test
@ -101,6 +104,9 @@ public void testImplicitDirectory() throws Exception {
assertEquals(fs.makeQualified(path), stat.getPath());
assertTrue(stat.isDirectory());
ContractTestUtils.assertNotErasureCoded(fs, path);
assertTrue(path + " should have erasure coding unset in " +
"FileStatus#toString(): " + stat,
stat.toString().contains("isErasureCoded=false"));
}
@Test

7
hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java
View File

@ -65,6 +65,9 @@ public void getFileStatusReturnsAsExpected()
Assert.assertEquals(new FsPermission("777"), fileStatus.getPermission());
Assert.assertEquals("NotSupportYet", fileStatus.getOwner());
Assert.assertEquals("NotSupportYet", fileStatus.getGroup());
Assert.assertTrue(path + " should have Acl!", fileStatus.hasAcl());
Assert.assertFalse(path + " should not be encrypted!",
fileStatus.isEncrypted());
Assert.assertFalse(path + " should not be erasure coded!",
fileStatus.isErasureCoded());
}
@ -82,6 +85,8 @@ public void getFileStatusAclBit()
LOG.debug("Time : " + (endTime - startTime));
Assert.assertTrue(fileStatus.isFile());
Assert.assertEquals(true, fileStatus.getPermission().getAclBit());
Assert.assertEquals(fileStatus.hasAcl(),
fileStatus.getPermission().getAclBit());
// With ACLBIT set to false
getMockServer().enqueue(new MockResponse().setResponseCode(200)
@ -93,5 +98,7 @@ public void getFileStatusAclBit()
LOG.debug("Time : " + (endTime - startTime));
Assert.assertTrue(fileStatus.isFile());
Assert.assertEquals(false, fileStatus.getPermission().getAclBit());
Assert.assertEquals(fileStatus.hasAcl(),
fileStatus.getPermission().getAclBit());
}
}