diff --git a/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml b/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
index de76afbcbc..4bafd8e022 100644
--- a/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
+++ b/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
@@ -323,6 +323,10 @@
+
+
+
+
+
+
+
+
+
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotInfo.java
index 3ddfc85638..676e8276f2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshotInfo.java
@@ -19,7 +19,7 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.FsPermissionProto;
+import org.apache.hadoop.hdfs.protocol.proto.AclProtos.FsPermissionProto;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
/**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/package-info.java
new file mode 100644
index 0000000000..6233024467
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocolPB;
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
index 4c9224908b..3b3368d8e1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
@@ -31,7 +31,6 @@
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
-import org.apache.hadoop.hdfs.protocol.FsPermissionExtension;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
@@ -47,6 +46,7 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Arrays;
+import java.util.EnumSet;
import static org.apache.hadoop.util.Time.now;
@@ -384,7 +384,6 @@ static HdfsFileStatus createFileStatusForEditLog(
* @param child for a directory listing of the iip, else null
* @param storagePolicy for the path or closest ancestor
* @param needLocation if block locations need to be included or not
- * @param includeStoragePolicy if storage policy should be returned
* @return a file status
* @throws java.io.IOException if any error occurs
*/
@@ -439,7 +438,19 @@ private static HdfsFileStatus createFileStatus(
int childrenNum = node.isDirectory() ?
node.asDirectory().getChildrenNum(snapshot) : 0;
+ EnumSet flags =
+ EnumSet.noneOf(HdfsFileStatus.Flags.class);
INodeAttributes nodeAttrs = fsd.getAttributes(iip);
+ boolean hasAcl = nodeAttrs.getAclFeature() != null;
+ if (hasAcl) {
+ flags.add(HdfsFileStatus.Flags.HAS_ACL);
+ }
+ if (isEncrypted) {
+ flags.add(HdfsFileStatus.Flags.HAS_CRYPT);
+ }
+ if (isErasureCoded) {
+ flags.add(HdfsFileStatus.Flags.HAS_EC);
+ }
return createFileStatus(
size,
node.isDirectory(),
@@ -447,7 +458,8 @@ private static HdfsFileStatus createFileStatus(
blocksize,
node.getModificationTime(snapshot),
node.getAccessTime(snapshot),
- getPermissionForFileStatus(nodeAttrs, isEncrypted, isErasureCoded),
+ nodeAttrs.getFsPermission(),
+ flags,
nodeAttrs.getUserName(),
nodeAttrs.getGroupName(),
node.isSymlink() ? node.asSymlink().getSymlink() : null,
@@ -460,44 +472,24 @@ private static HdfsFileStatus createFileStatus(
loc);
}
- private static HdfsFileStatus createFileStatus(long length, boolean isdir,
- int replication, long blocksize, long mtime,
- long atime, FsPermission permission, String owner, String group,
- byte[] symlink, byte[] path, long fileId, int childrenNum,
- FileEncryptionInfo feInfo, byte storagePolicy,
+ private static HdfsFileStatus createFileStatus(
+ long length, boolean isdir,
+ int replication, long blocksize, long mtime, long atime,
+ FsPermission permission, EnumSet flags,
+ String owner, String group, byte[] symlink, byte[] path, long fileId,
+ int childrenNum, FileEncryptionInfo feInfo, byte storagePolicy,
ErasureCodingPolicy ecPolicy, LocatedBlocks locations) {
if (locations == null) {
return new HdfsFileStatus(length, isdir, replication, blocksize,
- mtime, atime, permission, owner, group, symlink, path, fileId,
- childrenNum, feInfo, storagePolicy, ecPolicy);
+ mtime, atime, permission, flags, owner, group, symlink, path,
+ fileId, childrenNum, feInfo, storagePolicy, ecPolicy);
} else {
return new HdfsLocatedFileStatus(length, isdir, replication, blocksize,
- mtime, atime, permission, owner, group, symlink, path, fileId,
- locations, childrenNum, feInfo, storagePolicy, ecPolicy);
+ mtime, atime, permission, flags, owner, group, symlink, path,
+ fileId, locations, childrenNum, feInfo, storagePolicy, ecPolicy);
}
}
- /**
- * Returns an inode's FsPermission for use in an outbound FileStatus. If the
- * inode has an ACL or is for an encrypted file/dir, then this method will
- * return an FsPermissionExtension.
- *
- * @param node INode to check
- * @param isEncrypted boolean true if the file/dir is encrypted
- * @return FsPermission from inode, with ACL bit on if the inode has an ACL
- * and encrypted bit on if it represents an encrypted file/dir.
- */
- private static FsPermission getPermissionForFileStatus(
- INodeAttributes node, boolean isEncrypted, boolean isErasureCoded) {
- FsPermission perm = node.getFsPermission();
- boolean hasAcl = node.getAclFeature() != null;
- if (hasAcl || isEncrypted || isErasureCoded) {
- perm = new FsPermissionExtension(perm, hasAcl,
- isEncrypted, isErasureCoded);
- }
- return perm;
- }
-
private static ContentSummary getContentSummaryInt(FSDirectory fsd,
INodesInPath iip) throws IOException {
fsd.readLock();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 07dc5c1fa1..87b11562de 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -72,12 +72,13 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
-import java.util.concurrent.ForkJoinPool;
-import java.util.concurrent.RecursiveAction;
+import java.util.EnumSet;
import java.util.List;
import java.util.Map;
import java.util.SortedSet;
import java.util.TreeSet;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.RecursiveAction;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import static org.apache.hadoop.fs.CommonConfigurationKeys.FS_PROTECTED_DIRECTORIES;
@@ -135,11 +136,13 @@ private static INodeDirectory createRoot(FSNamesystem namesystem) {
public final static HdfsFileStatus DOT_RESERVED_STATUS =
new HdfsFileStatus(0, true, 0, 0, 0, 0, new FsPermission((short) 01770),
- null, null, null, HdfsFileStatus.EMPTY_NAME, -1L, 0, null,
+ EnumSet.noneOf(HdfsFileStatus.Flags.class), null, null, null,
+ HdfsFileStatus.EMPTY_NAME, -1L, 0, null,
HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, null);
public final static HdfsFileStatus DOT_SNAPSHOT_DIR_STATUS =
- new HdfsFileStatus(0, true, 0, 0, 0, 0, null, null, null, null,
+ new HdfsFileStatus(0, true, 0, 0, 0, 0, null,
+ EnumSet.noneOf(HdfsFileStatus.Flags.class), null, null, null,
HdfsFileStatus.EMPTY_NAME, -1L, 0, null,
HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, null);
@@ -383,12 +386,15 @@ HdfsFileStatus[] getReservedStatuses() {
*/
void createReservedStatuses(long cTime) {
HdfsFileStatus inodes = new HdfsFileStatus(0, true, 0, 0, cTime, cTime,
- new FsPermission((short) 0770), null, supergroup, null,
+ new FsPermission((short) 0770),
+ EnumSet.noneOf(HdfsFileStatus.Flags.class), null, supergroup, null,
DOT_INODES, -1L, 0, null,
HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, null);
HdfsFileStatus raw = new HdfsFileStatus(0, true, 0, 0, cTime, cTime,
- new FsPermission((short) 0770), null, supergroup, null, RAW, -1L,
- 0, null, HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, null);
+ new FsPermission((short) 0770),
+ EnumSet.noneOf(HdfsFileStatus.Flags.class), null, supergroup, null,
+ RAW, -1L, 0, null,
+ HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, null);
reservedStatuses = new HdfsFileStatus[] {inodes, raw};
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index f0ebcbba68..5849712203 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -177,6 +177,7 @@
import org.apache.hadoop.ha.ServiceFailedException;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.HAUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.UnknownCryptoProtocolVersionException;
@@ -371,9 +372,11 @@ private void logAuditEvent(boolean succeeded, String cmd, String src,
}
FileStatus status = null;
if (stat != null) {
- Path symlink = stat.isSymlink() ? new Path(stat.getSymlink()) : null;
+ Path symlink = stat.isSymlink()
+ ? new Path(DFSUtilClient.bytes2String(stat.getSymlinkInBytes()))
+ : null;
Path path = new Path(src);
- status = new FileStatus(stat.getLen(), stat.isDir(),
+ status = new FileStatus(stat.getLen(), stat.isDirectory(),
stat.getReplication(), stat.getBlockSize(),
stat.getModificationTime(),
stat.getAccessTime(), stat.getPermission(), stat.getOwner(),
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index 9cd58cb46a..52b422c935 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -80,6 +80,7 @@
import org.apache.hadoop.ha.protocolPB.HAServiceProtocolServerSideTranslatorPB;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.HDFSPolicyProvider;
import org.apache.hadoop.hdfs.inotify.EventBatch;
import org.apache.hadoop.hdfs.inotify.EventBatchList;
@@ -1430,7 +1431,7 @@ public String getLinkTarget(String path) throws IOException {
} else if (!stat.isSymlink()) {
throw new IOException("Path " + path + " is not a symbolic link");
}
- return stat.getSymlink();
+ return DFSUtilClient.bytes2String(stat.getSymlinkInBytes());
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
index ffc203f9d9..4b479e04d8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
@@ -25,6 +25,7 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -35,6 +36,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.DFSUtilClient;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocol.SnapshotException;
import org.apache.hadoop.hdfs.protocol.SnapshotInfo;
@@ -345,8 +347,9 @@ public SnapshottableDirectoryStatus[] getSnapshottableDirListing(
if (userName == null || userName.equals(dir.getUserName())) {
SnapshottableDirectoryStatus status = new SnapshottableDirectoryStatus(
dir.getModificationTime(), dir.getAccessTime(),
- dir.getFsPermission(), dir.getUserName(), dir.getGroupName(),
- dir.getLocalNameBytes(), dir.getId(),
+ dir.getFsPermission(), EnumSet.noneOf(HdfsFileStatus.Flags.class),
+ dir.getUserName(), dir.getGroupName(),
+ dir.getLocalNameBytes(), dir.getId(),
dir.getChildrenNum(Snapshot.CURRENT_STATE_ID),
dir.getDirectorySnapshottableFeature().getNumSnapshots(),
dir.getDirectorySnapshottableFeature().getSnapshotQuota(),
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
index a8861a8ccb..94752f5357 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
@@ -17,10 +17,18 @@
*/
package org.apache.hadoop.hdfs.web;
-import org.apache.hadoop.fs.*;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.FileChecksum;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.fs.XAttr;
+import org.apache.hadoop.fs.XAttrCodec;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.XAttrHelper;
import org.apache.hadoop.hdfs.protocol.*;
import org.apache.hadoop.ipc.RemoteException;
@@ -110,21 +118,20 @@ private static Map toJsonMap(HdfsFileStatus status) {
m.put("pathSuffix", status.getLocalName());
m.put("type", WebHdfsConstants.PathType.valueOf(status));
if (status.isSymlink()) {
- m.put("symlink", status.getSymlink());
+ m.put("symlink", DFSUtilClient.bytes2String(status.getSymlinkInBytes()));
}
-
m.put("length", status.getLen());
m.put("owner", status.getOwner());
m.put("group", status.getGroup());
FsPermission perm = status.getPermission();
m.put("permission", toString(perm));
- if (perm.getAclBit()) {
+ if (status.hasAcl()) {
m.put("aclBit", true);
}
- if (perm.getEncryptedBit()) {
+ if (status.isEncrypted()) {
m.put("encBit", true);
}
- if (perm.getErasureCodedBit()) {
+ if (status.isErasureCoded()) {
m.put("ecBit", true);
}
m.put("accessTime", status.getAccessTime());
@@ -373,15 +380,6 @@ public static String toJsonString(final AclStatus status) {
FsPermission perm = status.getPermission();
if (perm != null) {
m.put("permission", toString(perm));
- if (perm.getAclBit()) {
- m.put("aclBit", true);
- }
- if (perm.getEncryptedBit()) {
- m.put("encBit", true);
- }
- if (perm.getErasureCodedBit()) {
- m.put("ecBit", true);
- }
}
final Map> finalMap =
new TreeMap>();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
index 7a71df8d90..5b16f4c038 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
@@ -259,12 +259,14 @@ public Object answer(InvocationOnMock invocation)
Mockito.doReturn(
new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission(
- (short) 777), "owner", "group", new byte[0], new byte[0],
+ (short) 777), EnumSet.noneOf(HdfsFileStatus.Flags.class),
+ "owner", "group", new byte[0], new byte[0],
1010, 0, null, (byte) 0, null)).when(mockNN).getFileInfo(anyString());
Mockito.doReturn(
new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission(
- (short) 777), "owner", "group", new byte[0], new byte[0],
+ (short) 777), EnumSet.noneOf(HdfsFileStatus.Flags.class),
+ "owner", "group", new byte[0], new byte[0],
1010, 0, null, (byte) 0, null))
.when(mockNN)
.create(anyString(), (FsPermission) anyObject(), anyString(),
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index 8eb3b7b369..bf02db3a09 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -891,7 +891,8 @@ private static void mockCreate(ClientProtocol mcp,
CipherSuite suite, CryptoProtocolVersion version) throws Exception {
Mockito.doReturn(
new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission(
- (short) 777), "owner", "group", new byte[0], new byte[0],
+ (short) 777), EnumSet.noneOf(HdfsFileStatus.Flags.class),
+ "owner", "group", new byte[0], new byte[0],
1010, 0, new FileEncryptionInfo(suite,
version, new byte[suite.getAlgorithmBlockSize()],
new byte[suite.getAlgorithmBlockSize()],
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusSerialization.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusSerialization.java
new file mode 100644
index 0000000000..e5d05667f2
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusSerialization.java
@@ -0,0 +1,153 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import java.net.URI;
+
+import org.apache.hadoop.fs.FSProtos.FileStatusProto;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.HdfsFileStatusProto.FileType;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.HdfsFileStatusProto;
+import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+
+import com.google.protobuf.ByteString;
+
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Verify compatible FileStatus/HdfsFileStatus serialization.
+ */
+public class TestFileStatusSerialization {
+
+ private static void checkFields(FileStatus expected, FileStatus actual) {
+ assertEquals(expected.getPath(), actual.getPath());
+ assertEquals(expected.isDirectory(), actual.isDirectory());
+ assertEquals(expected.getLen(), actual.getLen());
+ assertEquals(expected.getPermission(), actual.getPermission());
+ assertEquals(expected.getOwner(), actual.getOwner());
+ assertEquals(expected.getGroup(), actual.getGroup());
+ assertEquals(expected.getModificationTime(), actual.getModificationTime());
+ assertEquals(expected.getAccessTime(), actual.getAccessTime());
+ assertEquals(expected.getReplication(), actual.getReplication());
+ assertEquals(expected.getBlockSize(), actual.getBlockSize());
+ }
+
+ /**
+ * Test API backwards-compatibility with 2.x applications w.r.t. FsPermission.
+ */
+ @Test
+ @SuppressWarnings("deprecation")
+ public void testFsPermissionCompatibility() throws Exception {
+ final int flagmask = 0x8;
+ // flags compatible with 2.x; fixed as constant in this test to ensure
+ // compatibility is maintained. New flags are not part of the contract this
+ // test verifies.
+ for (int i = 0; i < flagmask; ++i) {
+ FsPermission perm = FsPermission.createImmutable((short) 0013);
+ HdfsFileStatusProto.Builder hspb = HdfsFileStatusProto.newBuilder()
+ .setFileType(FileType.IS_FILE)
+ .setPath(ByteString.copyFromUtf8("hdfs://foobar/dingos/zot"))
+ .setLength(4344)
+ .setPermission(PBHelperClient.convert(perm))
+ .setOwner("hadoop")
+ .setGroup("unqbbc")
+ .setModificationTime(12345678L)
+ .setAccessTime(87654321L)
+ .setBlockReplication(10)
+ .setBlocksize(1L << 33)
+ .setFlags(i);
+ HdfsFileStatus stat = PBHelperClient.convert(hspb.build());
+ stat.makeQualified(new URI("hdfs://foobar"), new Path("/dingos"));
+ assertEquals(new Path("hdfs://foobar/dingos/zot"), stat.getPath());
+
+ // verify deprecated FsPermissionExtension methods
+ FsPermission sp = stat.getPermission();
+ assertEquals(sp.getAclBit(), stat.hasAcl());
+ assertEquals(sp.getEncryptedBit(), stat.isEncrypted());
+ assertEquals(sp.getErasureCodedBit(), stat.isErasureCoded());
+
+ // verify Writable contract
+ DataOutputBuffer dob = new DataOutputBuffer();
+ stat.write(dob);
+ DataInputBuffer dib = new DataInputBuffer();
+ dib.reset(dob.getData(), 0, dob.getLength());
+ FileStatus fstat = new FileStatus();
+ fstat.readFields(dib);
+ checkFields(stat, fstat);
+
+ // FsPermisisonExtension used for HdfsFileStatus, not FileStatus,
+ // attribute flags should still be preserved
+ assertEquals(sp.getAclBit(), fstat.hasAcl());
+ assertEquals(sp.getEncryptedBit(), fstat.isEncrypted());
+ assertEquals(sp.getErasureCodedBit(), fstat.isErasureCoded());
+ }
+ }
+ // param for LocatedFileStatus, HttpFileStatus
+
+ @Test
+ public void testCrossSerializationProto() throws Exception {
+ FsPermission perm = FsPermission.getFileDefault();
+ for (FileType t : FileType.values()) {
+ HdfsFileStatusProto.Builder hspb = HdfsFileStatusProto.newBuilder()
+ .setFileType(t)
+ .setPath(ByteString.copyFromUtf8("hdfs://foobar/dingos"))
+ .setLength(4344)
+ .setPermission(PBHelperClient.convert(perm))
+ .setOwner("hadoop")
+ .setGroup("unqbbc")
+ .setModificationTime(12345678L)
+ .setAccessTime(87654321L)
+ .setBlockReplication(10)
+ .setBlocksize(1L << 33);
+ if (FileType.IS_SYMLINK.equals(t)) {
+ hspb.setSymlink(ByteString.copyFromUtf8("hdfs://yaks/dingos"));
+ }
+ if (FileType.IS_FILE.equals(t)) {
+ hspb.setFileId(4544);
+ }
+ HdfsFileStatusProto hsp = hspb.build();
+ byte[] src = hsp.toByteArray();
+ FileStatusProto fsp = FileStatusProto.parseFrom(src);
+ assertEquals(hsp.getPath().toStringUtf8(), fsp.getPath());
+ assertEquals(hsp.getLength(), fsp.getLength());
+ assertEquals(hsp.getPermission().getPerm(),
+ fsp.getPermission().getPerm());
+ assertEquals(hsp.getOwner(), fsp.getOwner());
+ assertEquals(hsp.getGroup(), fsp.getGroup());
+ assertEquals(hsp.getModificationTime(), fsp.getModificationTime());
+ assertEquals(hsp.getAccessTime(), fsp.getAccessTime());
+ assertEquals(hsp.getSymlink().toStringUtf8(), fsp.getSymlink());
+ assertEquals(hsp.getBlockReplication(), fsp.getBlockReplication());
+ assertEquals(hsp.getBlocksize(), fsp.getBlockSize());
+ assertEquals(hsp.getFileType().ordinal(), fsp.getFileType().ordinal());
+
+ // verify unknown fields preserved
+ byte[] dst = fsp.toByteArray();
+ HdfsFileStatusProto hsp2 = HdfsFileStatusProto.parseFrom(dst);
+ assertEquals(hsp, hsp2);
+ checkFields(PBHelperClient.convert(hsp), PBHelperClient.convert(hsp2));
+ }
+ }
+
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java
index 16cdf9b322..161e2277ce 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java
@@ -30,6 +30,7 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
+import java.util.EnumSet;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -353,12 +354,14 @@ public void testFactory() throws Exception {
Mockito.doReturn(
new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission(
- (short) 777), "owner", "group", new byte[0], new byte[0],
+ (short) 777), EnumSet.noneOf(HdfsFileStatus.Flags.class),
+ "owner", "group", new byte[0], new byte[0],
1010, 0, null, (byte) 0, null)).when(mcp).getFileInfo(anyString());
Mockito
.doReturn(
new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission(
- (short) 777), "owner", "group", new byte[0], new byte[0],
+ (short) 777), EnumSet.noneOf(HdfsFileStatus.Flags.class),
+ "owner", "group", new byte[0], new byte[0],
1010, 0, null, (byte) 0, null))
.when(mcp)
.create(anyString(), (FsPermission) anyObject(), anyString(),
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/AclTestHelpers.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/AclTestHelpers.java
index 52a68584dd..646e80083d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/AclTestHelpers.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/AclTestHelpers.java
@@ -21,6 +21,7 @@
import java.io.IOException;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.AclEntry;
@@ -141,6 +142,11 @@ public static void assertFilePermissionGranted(FileSystem fs,
}
}
+ public static void assertPermission(FileSystem fs, Path pathToCheck,
+ short perm) throws IOException {
+ assertPermission(fs, pathToCheck, perm, (perm & (1 << 12)) != 0);
+ }
+
/**
* Asserts the value of the FsPermission bits on the inode of a specific path.
*
@@ -150,10 +156,11 @@ public static void assertFilePermissionGranted(FileSystem fs,
* @throws IOException thrown if there is an I/O error
*/
public static void assertPermission(FileSystem fs, Path pathToCheck,
- short perm) throws IOException {
+ short perm, boolean hasAcl) throws IOException {
short filteredPerm = (short)(perm & 01777);
- FsPermission fsPermission = fs.getFileStatus(pathToCheck).getPermission();
+ FileStatus stat = fs.getFileStatus(pathToCheck);
+ FsPermission fsPermission = stat.getPermission();
assertEquals(filteredPerm, fsPermission.toShort());
- assertEquals(((perm & (1 << 12)) != 0), fsPermission.getAclBit());
+ assertEquals(hasAcl, stat.hasAcl());
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
index 6c755e78ec..38c17b7154 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
@@ -48,6 +48,7 @@
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -1355,7 +1356,8 @@ public void testFsckFileNotFound() throws Exception {
byte storagePolicy = 0;
HdfsFileStatus file = new HdfsFileStatus(length, isDir, blockReplication,
- blockSize, modTime, accessTime, perms, owner, group, symlink,
+ blockSize, modTime, accessTime, perms,
+ EnumSet.noneOf(HdfsFileStatus.Flags.class), owner, group, symlink,
path, fileId, numChildren, null, storagePolicy, null);
Result replRes = new ReplicationResult(conf);
Result ecRes = new ErasureCodingResult(conf);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java
index 5a3d451bef..edb79d3bb4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java
@@ -23,6 +23,7 @@
import static org.apache.hadoop.hdfs.server.namenode.AclTestHelpers.*;
import java.io.IOException;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -37,6 +38,7 @@
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.XAttrHelper;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -51,10 +53,12 @@
public class TestJsonUtil {
static FileStatus toFileStatus(HdfsFileStatus f, String parent) {
- return new FileStatus(f.getLen(), f.isDir(), f.getReplication(),
+ return new FileStatus(f.getLen(), f.isDirectory(), f.getReplication(),
f.getBlockSize(), f.getModificationTime(), f.getAccessTime(),
f.getPermission(), f.getOwner(), f.getGroup(),
- f.isSymlink() ? new Path(f.getSymlink()) : null,
+ f.isSymlink()
+ ? new Path(DFSUtilClient.bytes2String(f.getSymlinkInBytes()))
+ : null,
new Path(f.getFullName(parent)));
}
@@ -63,7 +67,8 @@ public void testHdfsFileStatus() throws IOException {
final long now = Time.now();
final String parent = "/dir";
final HdfsFileStatus status = new HdfsFileStatus(1001L, false, 3, 1L << 26,
- now, now + 10, new FsPermission((short) 0644), "user", "group",
+ now, now + 10, new FsPermission((short) 0644),
+ EnumSet.noneOf(HdfsFileStatus.Flags.class), "user", "group",
DFSUtil.string2Bytes("bar"), DFSUtil.string2Bytes("foo"),
HdfsConstants.GRANDFATHER_INODE_ID, 0, null, (byte) 0, null);
final FileStatus fstatus = toFileStatus(status, parent);