big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-11762. [SPS]: Empty files should be ignored in StoragePolicySatisfier. Surendra Singh Lilhore.

Browse Source
This commit is contained in:
Rakesh Radhakrishnan 2017-06-05 12:32:41 +05:30 committed by Uma Maheswara Rao Gangumalla
parent e53f89ccc3
commit d638a7dc03
2 changed files with 42 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSatisfyStoragePolicyOp.java
View File

@ -51,7 +51,6 @@ static FileStatus satisfyStoragePolicy(FSDirectory fsd, BlockManager bm,
assert fsd.getFSNamesystem().hasWriteLock(); assert fsd.getFSNamesystem().hasWriteLock();
FSPermissionChecker pc = fsd.getPermissionChecker(); FSPermissionChecker pc = fsd.getPermissionChecker();
List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
INodesInPath iip; INodesInPath iip;
fsd.writeLock(); fsd.writeLock();
try { try {
@ -62,8 +61,11 @@ static FileStatus satisfyStoragePolicy(FSDirectory fsd, BlockManager bm,
fsd.checkPathAccess(pc, iip, FsAction.WRITE); fsd.checkPathAccess(pc, iip, FsAction.WRITE);
} }
XAttr satisfyXAttr = unprotectedSatisfyStoragePolicy(iip, bm, fsd); XAttr satisfyXAttr = unprotectedSatisfyStoragePolicy(iip, bm, fsd);
if (satisfyXAttr != null) {
List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
xAttrs.add(satisfyXAttr); xAttrs.add(satisfyXAttr);
fsd.getEditLog().logSetXAttrs(src, xAttrs, logRetryCache); fsd.getEditLog().logSetXAttrs(src, xAttrs, logRetryCache);
}
} finally { } finally {
fsd.writeUnlock(); fsd.writeUnlock();
} }
@ -79,16 +81,19 @@ static XAttr unprotectedSatisfyStoragePolicy(INodesInPath iip,
// TODO: think about optimization here, label the dir instead // TODO: think about optimization here, label the dir instead
// of the sub-files of the dir. // of the sub-files of the dir.
if (inode.isFile()) { if (inode.isFile() && inode.asFile().numBlocks() != 0) {
candidateNodes.add(inode); candidateNodes.add(inode);
} else if (inode.isDirectory()) { } else if (inode.isDirectory()) {
for (INode node : inode.asDirectory().getChildrenList(snapshotId)) { for (INode node : inode.asDirectory().getChildrenList(snapshotId)) {
if (node.isFile()) { if (node.isFile() && node.asFile().numBlocks() != 0) {
candidateNodes.add(node); candidateNodes.add(node);
} }
} }
} }
if (candidateNodes.isEmpty()) {
return null;
}
// If node has satisfy xattr, then stop adding it // If node has satisfy xattr, then stop adding it
// to satisfy movement queue. // to satisfy movement queue.
if (inodeHasSatisfyXAttr(candidateNodes)) { if (inodeHasSatisfyXAttr(candidateNodes)) {

32
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStoragePolicySatisfier.java
View File

@ -907,6 +907,38 @@ public void testSPSShouldNotLeakXattrIfSatisfyStoragePolicyCallOnECFiles()
} }
} }
/**
* Test SPS with empty file.
* 1. Create one empty file.
* 2. Call satisfyStoragePolicy for empty file.
* 3. SPS should skip this file and xattr should not be added for empty file.
*/
@Test(timeout = 300000)
public void testSPSWhenFileLengthIsZero() throws Exception {
MiniDFSCluster cluster = null;
try {
cluster = new MiniDFSCluster.Builder(new Configuration()).numDataNodes(0)
.build();
cluster.waitActive();
DistributedFileSystem fs = cluster.getFileSystem();
Path filePath = new Path("/zeroSizeFile");
DFSTestUtil.createFile(fs, filePath, 0, (short) 1, 0);
FSEditLog editlog = cluster.getNameNode().getNamesystem().getEditLog();
long lastWrittenTxId = editlog.getLastWrittenTxId();
fs.satisfyStoragePolicy(filePath);
Assert.assertEquals("Xattr should not be added for the file",
lastWrittenTxId, editlog.getLastWrittenTxId());
INode inode = cluster.getNameNode().getNamesystem().getFSDirectory()
.getINode(filePath.toString());
Assert.assertTrue("XAttrFeature should be null for file",
inode.getXAttrFeature() == null);
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
private String createFileAndSimulateFavoredNodes(int favoredNodesCount) private String createFileAndSimulateFavoredNodes(int favoredNodesCount)
throws IOException { throws IOException {
ArrayList<DataNode> dns = hdfsCluster.getDataNodes(); ArrayList<DataNode> dns = hdfsCluster.getDataNodes();