HDFS-11762. [SPS]: Empty files should be ignored in StoragePolicySatisfier. Surendra Singh Lilhore.
This commit is contained in:
parent
e53f89ccc3
commit
d638a7dc03
@ -51,7 +51,6 @@ static FileStatus satisfyStoragePolicy(FSDirectory fsd, BlockManager bm,
|
|||||||
|
|
||||||
assert fsd.getFSNamesystem().hasWriteLock();
|
assert fsd.getFSNamesystem().hasWriteLock();
|
||||||
FSPermissionChecker pc = fsd.getPermissionChecker();
|
FSPermissionChecker pc = fsd.getPermissionChecker();
|
||||||
List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
|
|
||||||
INodesInPath iip;
|
INodesInPath iip;
|
||||||
fsd.writeLock();
|
fsd.writeLock();
|
||||||
try {
|
try {
|
||||||
@ -62,8 +61,11 @@ static FileStatus satisfyStoragePolicy(FSDirectory fsd, BlockManager bm,
|
|||||||
fsd.checkPathAccess(pc, iip, FsAction.WRITE);
|
fsd.checkPathAccess(pc, iip, FsAction.WRITE);
|
||||||
}
|
}
|
||||||
XAttr satisfyXAttr = unprotectedSatisfyStoragePolicy(iip, bm, fsd);
|
XAttr satisfyXAttr = unprotectedSatisfyStoragePolicy(iip, bm, fsd);
|
||||||
xAttrs.add(satisfyXAttr);
|
if (satisfyXAttr != null) {
|
||||||
fsd.getEditLog().logSetXAttrs(src, xAttrs, logRetryCache);
|
List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
|
||||||
|
xAttrs.add(satisfyXAttr);
|
||||||
|
fsd.getEditLog().logSetXAttrs(src, xAttrs, logRetryCache);
|
||||||
|
}
|
||||||
} finally {
|
} finally {
|
||||||
fsd.writeUnlock();
|
fsd.writeUnlock();
|
||||||
}
|
}
|
||||||
@ -79,16 +81,19 @@ static XAttr unprotectedSatisfyStoragePolicy(INodesInPath iip,
|
|||||||
|
|
||||||
// TODO: think about optimization here, label the dir instead
|
// TODO: think about optimization here, label the dir instead
|
||||||
// of the sub-files of the dir.
|
// of the sub-files of the dir.
|
||||||
if (inode.isFile()) {
|
if (inode.isFile() && inode.asFile().numBlocks() != 0) {
|
||||||
candidateNodes.add(inode);
|
candidateNodes.add(inode);
|
||||||
} else if (inode.isDirectory()) {
|
} else if (inode.isDirectory()) {
|
||||||
for (INode node : inode.asDirectory().getChildrenList(snapshotId)) {
|
for (INode node : inode.asDirectory().getChildrenList(snapshotId)) {
|
||||||
if (node.isFile()) {
|
if (node.isFile() && node.asFile().numBlocks() != 0) {
|
||||||
candidateNodes.add(node);
|
candidateNodes.add(node);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (candidateNodes.isEmpty()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
// If node has satisfy xattr, then stop adding it
|
// If node has satisfy xattr, then stop adding it
|
||||||
// to satisfy movement queue.
|
// to satisfy movement queue.
|
||||||
if (inodeHasSatisfyXAttr(candidateNodes)) {
|
if (inodeHasSatisfyXAttr(candidateNodes)) {
|
||||||
|
@ -907,6 +907,38 @@ public void testSPSShouldNotLeakXattrIfSatisfyStoragePolicyCallOnECFiles()
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test SPS with empty file.
|
||||||
|
* 1. Create one empty file.
|
||||||
|
* 2. Call satisfyStoragePolicy for empty file.
|
||||||
|
* 3. SPS should skip this file and xattr should not be added for empty file.
|
||||||
|
*/
|
||||||
|
@Test(timeout = 300000)
|
||||||
|
public void testSPSWhenFileLengthIsZero() throws Exception {
|
||||||
|
MiniDFSCluster cluster = null;
|
||||||
|
try {
|
||||||
|
cluster = new MiniDFSCluster.Builder(new Configuration()).numDataNodes(0)
|
||||||
|
.build();
|
||||||
|
cluster.waitActive();
|
||||||
|
DistributedFileSystem fs = cluster.getFileSystem();
|
||||||
|
Path filePath = new Path("/zeroSizeFile");
|
||||||
|
DFSTestUtil.createFile(fs, filePath, 0, (short) 1, 0);
|
||||||
|
FSEditLog editlog = cluster.getNameNode().getNamesystem().getEditLog();
|
||||||
|
long lastWrittenTxId = editlog.getLastWrittenTxId();
|
||||||
|
fs.satisfyStoragePolicy(filePath);
|
||||||
|
Assert.assertEquals("Xattr should not be added for the file",
|
||||||
|
lastWrittenTxId, editlog.getLastWrittenTxId());
|
||||||
|
INode inode = cluster.getNameNode().getNamesystem().getFSDirectory()
|
||||||
|
.getINode(filePath.toString());
|
||||||
|
Assert.assertTrue("XAttrFeature should be null for file",
|
||||||
|
inode.getXAttrFeature() == null);
|
||||||
|
} finally {
|
||||||
|
if (cluster != null) {
|
||||||
|
cluster.shutdown();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private String createFileAndSimulateFavoredNodes(int favoredNodesCount)
|
private String createFileAndSimulateFavoredNodes(int favoredNodesCount)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
ArrayList<DataNode> dns = hdfsCluster.getDataNodes();
|
ArrayList<DataNode> dns = hdfsCluster.getDataNodes();
|
||||||
|
Loading…
Reference in New Issue
Block a user