big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-8200. Refactor FSDirStatAndListingOp. Contributed by Haohui Mai.

Browse Source
This commit is contained in:
Haohui Mai 2015-04-30 13:41:46 -07:00
parent 7e8639fda4
commit c55d609053
3 changed files with 87 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -486,6 +486,8 @@ Release 2.8.0 - UNRELEASED
HDFS-5574. Remove buffer copy in BlockReader.skip. HDFS-5574. Remove buffer copy in BlockReader.skip.
(Binglin Chang via aajisaka) (Binglin Chang via aajisaka)
HDFS-8200. Refactor FSDirStatAndListingOp. (wheat9)
OPTIMIZATIONS OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than

107
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
View File

@ -24,7 +24,6 @@
import org.apache.hadoop.fs.DirectoryListingStartAfterNotFoundException; import org.apache.hadoop.fs.DirectoryListingStartAfterNotFoundException;
import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.fs.InvalidPathException; import org.apache.hadoop.fs.InvalidPathException;
import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.DFSUtil;
@ -180,10 +179,14 @@ private static DirectoryListing getListing(FSDirectory fsd, INodesInPath iip,
.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED; .BLOCK_STORAGE_POLICY_ID_UNSPECIFIED;
if (!targetNode.isDirectory()) { if (!targetNode.isDirectory()) {
INodeAttributes nodeAttrs = getINodeAttributes(
fsd, src, HdfsFileStatus.EMPTY_NAME, targetNode,
snapshot);
return new DirectoryListing( return new DirectoryListing(
new HdfsFileStatus[]{createFileStatus(fsd, src, new HdfsFileStatus[]{ createFileStatus(
HdfsFileStatus.EMPTY_NAME, targetNode, needLocation, fsd, HdfsFileStatus.EMPTY_NAME, targetNode, nodeAttrs,
parentStoragePolicy, snapshot, isRawPath, iip)}, 0); needLocation, parentStoragePolicy, snapshot, isRawPath, iip)
}, 0);
} }
final INodeDirectory dirInode = targetNode.asDirectory(); final INodeDirectory dirInode = targetNode.asDirectory();
@ -200,8 +203,11 @@ private static DirectoryListing getListing(FSDirectory fsd, INodesInPath iip,
byte curPolicy = isSuperUser && !cur.isSymlink()? byte curPolicy = isSuperUser && !cur.isSymlink()?
cur.getLocalStoragePolicyID(): cur.getLocalStoragePolicyID():
HdfsConstantsClient.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED; HdfsConstantsClient.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED;
listing[i] = createFileStatus(fsd, src, cur.getLocalNameBytes(), cur, INodeAttributes nodeAttrs = getINodeAttributes(
needLocation, getStoragePolicyID(curPolicy, fsd, src, cur.getLocalNameBytes(), cur,
snapshot);
listing[i] = createFileStatus(fsd, cur.getLocalNameBytes(),
cur, nodeAttrs, needLocation, getStoragePolicyID(curPolicy,
parentStoragePolicy), snapshot, isRawPath, iip); parentStoragePolicy), snapshot, isRawPath, iip);
listingCnt++; listingCnt++;
if (needLocation) { if (needLocation) {
@ -253,9 +259,15 @@ private static DirectoryListing getSnapshotsListing(
final HdfsFileStatus listing[] = new HdfsFileStatus[numOfListing]; final HdfsFileStatus listing[] = new HdfsFileStatus[numOfListing];
for (int i = 0; i < numOfListing; i++) { for (int i = 0; i < numOfListing; i++) {
Snapshot.Root sRoot = snapshots.get(i + skipSize).getRoot(); Snapshot.Root sRoot = snapshots.get(i + skipSize).getRoot();
listing[i] = createFileStatus(fsd, src, sRoot.getLocalNameBytes(), sRoot, INodeAttributes nodeAttrs = getINodeAttributes(
HdfsConstantsClient.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, Snapshot.CURRENT_STATE_ID, fsd, src, sRoot.getLocalNameBytes(),
false, INodesInPath.fromINode(sRoot)); node, Snapshot.CURRENT_STATE_ID);
listing[i] = createFileStatus(
fsd, sRoot.getLocalNameBytes(),
sRoot, nodeAttrs,
HdfsConstantsClient.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED,
Snapshot.CURRENT_STATE_ID, false,
INodesInPath.fromINode(sRoot));
} }
return new DirectoryListing( return new DirectoryListing(
listing, snapshots.size() - skipSize - numOfListing); listing, snapshots.size() - skipSize - numOfListing);
@ -276,11 +288,20 @@ static HdfsFileStatus getFileInfo(
fsd.readLock(); fsd.readLock();
try { try {
final INode i = src.getLastINode(); final INode i = src.getLastINode();
byte policyId = includeStoragePolicy && i != null && !i.isSymlink() ? if (i == null) {
i.getStoragePolicyID() : HdfsConstantsClient.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED; return null;
return i == null ? null : createFileStatus( }
fsd, path, HdfsFileStatus.EMPTY_NAME, i, policyId,
src.getPathSnapshotId(), isRawPath, src); byte policyId = includeStoragePolicy && !i.isSymlink() ?
i.getStoragePolicyID() : HdfsConstantsClient
.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED;
INodeAttributes nodeAttrs = getINodeAttributes(
fsd, path, HdfsFileStatus.EMPTY_NAME, i, src.getPathSnapshotId());
return createFileStatus(
fsd, HdfsFileStatus.EMPTY_NAME,
i, nodeAttrs, policyId,
src.getPathSnapshotId(),
isRawPath, src);
} finally { } finally {
fsd.readUnlock(); fsd.readUnlock();
} }
@ -309,23 +330,6 @@ static HdfsFileStatus getFileInfo(
} }
} }
/**
* Currently we only support "ls /xxx/.snapshot" which will return all the
* snapshots of a directory. The FSCommand Ls will first call getFileInfo to
* make sure the file/directory exists (before the real getListing call).
* Since we do not have a real INode for ".snapshot", we return an empty
* non-null HdfsFileStatus here.
*/
private static HdfsFileStatus getFileInfo4DotSnapshot(
FSDirectory fsd, String src)
throws UnresolvedLinkException {
if (fsd.getINode4DotSnapshot(src) != null) {
return new HdfsFileStatus(0, true, 0, 0, 0, 0, null, null, null, null,
HdfsFileStatus.EMPTY_NAME, -1L, 0, null,
HdfsConstantsClient.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED);
}
return null;
}
/** /**
* create an hdfs file status from an inode * create an hdfs file status from an inode
@ -339,27 +343,40 @@ private static HdfsFileStatus getFileInfo4DotSnapshot(
* @return a file status * @return a file status
* @throws java.io.IOException if any error occurs * @throws java.io.IOException if any error occurs
*/ */
static HdfsFileStatus createFileStatus( private static HdfsFileStatus createFileStatus(
FSDirectory fsd, String fullPath, byte[] path, INode node, FSDirectory fsd, byte[] path, INode node, INodeAttributes nodeAttrs,
boolean needLocation, byte storagePolicy, int snapshot, boolean isRawPath, boolean needLocation, byte storagePolicy, int snapshot, boolean isRawPath,
INodesInPath iip) INodesInPath iip)
throws IOException { throws IOException {
if (needLocation) { if (needLocation) {
return createLocatedFileStatus(fsd, fullPath, path, node, storagePolicy, return createLocatedFileStatus(fsd, path, node, nodeAttrs, storagePolicy,
snapshot, isRawPath, iip); snapshot, isRawPath, iip);
} else { } else {
return createFileStatus(fsd, fullPath, path, node, storagePolicy, snapshot, return createFileStatus(fsd, path, node, nodeAttrs, storagePolicy,
isRawPath, iip); snapshot, isRawPath, iip);
} }
} }
/** /**
* Create FileStatus by file INode * Create FileStatus by file INode
*/ */
static HdfsFileStatus createFileStatus( static HdfsFileStatus createFileStatusForEditLog(
FSDirectory fsd, String fullPath, byte[] path, INode node, FSDirectory fsd, String fullPath, byte[] path, INode node,
byte storagePolicy, int snapshot, boolean isRawPath, byte storagePolicy, int snapshot, boolean isRawPath,
INodesInPath iip) throws IOException { INodesInPath iip) throws IOException {
INodeAttributes nodeAttrs = getINodeAttributes(
fsd, fullPath, path, node, snapshot);
return createFileStatus(fsd, path, node, nodeAttrs,
storagePolicy, snapshot, isRawPath, iip);
}
/**
* Create FileStatus by file INode
*/
static HdfsFileStatus createFileStatus(
FSDirectory fsd, byte[] path, INode node,
INodeAttributes nodeAttrs, byte storagePolicy, int snapshot,
boolean isRawPath, INodesInPath iip) throws IOException {
long size = 0; // length is zero for directories long size = 0; // length is zero for directories
short replication = 0; short replication = 0;
long blocksize = 0; long blocksize = 0;
@ -382,8 +399,6 @@ static HdfsFileStatus createFileStatus(
int childrenNum = node.isDirectory() ? int childrenNum = node.isDirectory() ?
node.asDirectory().getChildrenNum(snapshot) : 0; node.asDirectory().getChildrenNum(snapshot) : 0;
INodeAttributes nodeAttrs =
fsd.getAttributes(fullPath, path, node, snapshot);
return new HdfsFileStatus( return new HdfsFileStatus(
size, size,
node.isDirectory(), node.isDirectory(),
@ -402,13 +417,18 @@ static HdfsFileStatus createFileStatus(
storagePolicy); storagePolicy);
} }
private static INodeAttributes getINodeAttributes(
FSDirectory fsd, String fullPath, byte[] path, INode node, int snapshot) {
return fsd.getAttributes(fullPath, path, node, snapshot);
}
/** /**
* Create FileStatus with location info by file INode * Create FileStatus with location info by file INode
*/ */
private static HdfsLocatedFileStatus createLocatedFileStatus( private static HdfsLocatedFileStatus createLocatedFileStatus(
FSDirectory fsd, String fullPath, byte[] path, INode node, FSDirectory fsd, byte[] path, INode node, INodeAttributes nodeAttrs,
byte storagePolicy, int snapshot, boolean isRawPath, byte storagePolicy, int snapshot,
INodesInPath iip) throws IOException { boolean isRawPath, INodesInPath iip) throws IOException {
assert fsd.hasReadLock(); assert fsd.hasReadLock();
long size = 0; // length is zero for directories long size = 0; // length is zero for directories
short replication = 0; short replication = 0;
@ -442,8 +462,6 @@ private static HdfsLocatedFileStatus createLocatedFileStatus(
int childrenNum = node.isDirectory() ? int childrenNum = node.isDirectory() ?
node.asDirectory().getChildrenNum(snapshot) : 0; node.asDirectory().getChildrenNum(snapshot) : 0;
INodeAttributes nodeAttrs =
fsd.getAttributes(fullPath, path, node, snapshot);
HdfsLocatedFileStatus status = HdfsLocatedFileStatus status =
new HdfsLocatedFileStatus(size, node.isDirectory(), replication, new HdfsLocatedFileStatus(size, node.isDirectory(), replication,
blocksize, node.getModificationTime(snapshot), blocksize, node.getModificationTime(snapshot),
@ -468,7 +486,6 @@ private static HdfsLocatedFileStatus createLocatedFileStatus(
* return an FsPermissionExtension. * return an FsPermissionExtension.
* *
* @param node INode to check * @param node INode to check
* @param snapshot int snapshot ID
* @param isEncrypted boolean true if the file/dir is encrypted * @param isEncrypted boolean true if the file/dir is encrypted
* @return FsPermission from inode, with ACL bit on if the inode has an ACL * @return FsPermission from inode, with ACL bit on if the inode has an ACL
* and encrypted bit on if it represents an encrypted file/dir. * and encrypted bit on if it represents an encrypted file/dir.

6
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
View File

@ -378,7 +378,7 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
// add the op into retry cache if necessary // add the op into retry cache if necessary
if (toAddRetryCache) { if (toAddRetryCache) {
HdfsFileStatus stat = FSDirStatAndListingOp.createFileStatus( HdfsFileStatus stat = FSDirStatAndListingOp.createFileStatusForEditLog(
fsNamesys.dir, path, HdfsFileStatus.EMPTY_NAME, newFile, fsNamesys.dir, path, HdfsFileStatus.EMPTY_NAME, newFile,
HdfsConstantsClient.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, Snapshot.CURRENT_STATE_ID, HdfsConstantsClient.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, Snapshot.CURRENT_STATE_ID,
false, iip); false, iip);
@ -397,7 +397,7 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
false); false);
// add the op into retry cache if necessary // add the op into retry cache if necessary
if (toAddRetryCache) { if (toAddRetryCache) {
HdfsFileStatus stat = FSDirStatAndListingOp.createFileStatus( HdfsFileStatus stat = FSDirStatAndListingOp.createFileStatusForEditLog(
fsNamesys.dir, path, fsNamesys.dir, path,
HdfsFileStatus.EMPTY_NAME, newFile, HdfsFileStatus.EMPTY_NAME, newFile,
HdfsConstantsClient.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, HdfsConstantsClient.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED,
@ -471,7 +471,7 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
false, false); false, false);
// add the op into retry cache if necessary // add the op into retry cache if necessary
if (toAddRetryCache) { if (toAddRetryCache) {
HdfsFileStatus stat = FSDirStatAndListingOp.createFileStatus( HdfsFileStatus stat = FSDirStatAndListingOp.createFileStatusForEditLog(
fsNamesys.dir, path, HdfsFileStatus.EMPTY_NAME, file, fsNamesys.dir, path, HdfsFileStatus.EMPTY_NAME, file,
HdfsConstantsClient.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, HdfsConstantsClient.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED,
Snapshot.CURRENT_STATE_ID, false, iip); Snapshot.CURRENT_STATE_ID, false, iip);