big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-6099. HDFS file system limits not enforced on renames. Contributed by Chris Nauroth.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1579122 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Chris Nauroth 2014-03-19 03:46:52 +00:00
parent d0f7985f5c
commit 2b03ae9421
4 changed files with 142 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -647,6 +647,8 @@ Release 2.4.0 - UNRELEASED
HDFS-6117. Print file path information in FileNotFoundException on INode
ID mismatch. (suresh)
HDFS-6099. HDFS file system limits not enforced on renames. (cnauroth)
BREAKDOWN OF HDFS-5698 SUBTASKS AND RELATED JIRAS
HDFS-5717. Save FSImage header in protobuf. (Haohui Mai via jing9)

36
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
View File

@ -643,6 +643,7 @@ boolean unprotectedRenameTo(String src, String dst, long timestamp)
}
// Ensure dst has quota to accommodate rename
verifyFsLimitsForRename(srcIIP, dstIIP);
verifyQuotaForRename(srcIIP.getINodes(), dstIIP.getINodes());
boolean added = false;
@ -894,6 +895,7 @@ boolean unprotectedRenameTo(String src, String dst, long timestamp,
}
// Ensure dst has quota to accommodate rename
verifyFsLimitsForRename(srcIIP, dstIIP);
verifyQuotaForRename(srcIIP.getINodes(), dstIIP.getINodes());
INode srcChild = srcIIP.getLastINode();
@ -2134,6 +2136,27 @@ private void verifyQuotaForRename(INode[] src, INode[] dst)
delta.get(Quota.DISKSPACE), src[i - 1]);
}
/**
* Checks file system limits (max component length and max directory items)
* during a rename operation.
*
* @param srcIIP INodesInPath containing every inode in the rename source
* @param dstIIP INodesInPath containing every inode in the rename destination
* @throws PathComponentTooLongException child's name is too long.
* @throws MaxDirectoryItemsExceededException too many children.
*/
private void verifyFsLimitsForRename(INodesInPath srcIIP, INodesInPath dstIIP)
throws PathComponentTooLongException, MaxDirectoryItemsExceededException {
byte[] dstChildName = dstIIP.getLastLocalName();
INode[] dstInodes = dstIIP.getINodes();
int pos = dstInodes.length - 1;
verifyMaxComponentLength(dstChildName, dstInodes, pos);
// Do not enforce max directory items if renaming within same directory.
if (srcIIP.getINode(-2) != dstIIP.getINode(-2)) {
verifyMaxDirItems(dstInodes, pos);
}
}
/** Verify if the snapshot name is legal. */
void verifySnapshotName(String snapshotName, String path)
throws PathComponentTooLongException {
@ -2159,10 +2182,14 @@ void verifyINodeName(byte[] childName) throws HadoopIllegalArgumentException {
/**
* Verify child's name for fs limit.
*
* @param childName byte[] containing new child name
* @param parentPath Object either INode[] or String containing parent path
* @param pos int position of new child in path
* @throws PathComponentTooLongException child's name is too long.
*/
void verifyMaxComponentLength(byte[] childName, Object parentPath, int pos)
throws PathComponentTooLongException {
private void verifyMaxComponentLength(byte[] childName, Object parentPath,
int pos) throws PathComponentTooLongException {
if (maxComponentLength == 0) {
return;
}
@ -2184,9 +2211,12 @@ void verifyMaxComponentLength(byte[] childName, Object parentPath, int pos)
/**
* Verify children size for fs limit.
*
* @param pathComponents INode[] containing full path of inodes to new child
* @param pos int position of new child in pathComponents
* @throws MaxDirectoryItemsExceededException too many children.
*/
void verifyMaxDirItems(INode[] pathComponents, int pos)
private void verifyMaxDirItems(INode[] pathComponents, int pos)
throws MaxDirectoryItemsExceededException {
final INodeDirectory parent = pathComponents[pos-1].asDirectory();

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
View File

@ -75,6 +75,7 @@
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.FSLimitException;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
@ -358,7 +359,9 @@ public NameNodeRpcServer(Configuration conf, NameNode nn)
LeaseExpiredException.class,
NSQuotaExceededException.class,
DSQuotaExceededException.class,
AclException.class);
AclException.class,
FSLimitException.PathComponentTooLongException.class,
FSLimitException.MaxDirectoryItemsExceededException.class);
}
/** Allow access to the client RPC server for testing */

146
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsLimits.java
View File

@ -19,6 +19,7 @@
package org.apache.hadoop.hdfs.server.namenode;
import static org.apache.hadoop.hdfs.server.common.Util.fileAsURI;
import static org.apache.hadoop.util.Time.now;
import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.anyObject;
import static org.mockito.Mockito.mock;
@ -29,14 +30,15 @@
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.FSLimitException.MaxDirectoryItemsExceededException;
import org.apache.hadoop.hdfs.protocol.FSLimitException.PathComponentTooLongException;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Before;
import org.junit.Test;
@ -50,7 +52,12 @@ public class TestFsLimits {
static PermissionStatus perms
= new PermissionStatus("admin", "admin", FsPermission.getDefault());
static INodeDirectory rootInode;
static private FSImage getMockFSImage() {
FSEditLog editLog = mock(FSEditLog.class);
FSImage fsImage = mock(FSImage.class);
when(fsImage.getEditLog()).thenReturn(editLog);
return fsImage;
}
static private FSNamesystem getMockNamesystem() {
FSNamesystem fsn = mock(FSNamesystem.class);
@ -64,8 +71,9 @@ static private FSNamesystem getMockNamesystem() {
private static class MockFSDirectory extends FSDirectory {
public MockFSDirectory() throws IOException {
super(new FSImage(conf), getMockNamesystem(), conf);
super(getMockFSImage(), getMockNamesystem(), conf);
setReady(fsIsReady);
NameNode.initMetrics(conf, NamenodeRole.NAMENODE);
}
}
@ -76,21 +84,18 @@ public void setUp() throws IOException {
fileAsURI(new File(MiniDFSCluster.getBaseDirectory(),
"namenode")).toString());
rootInode = new INodeDirectory(getMockNamesystem().allocateNewInodeId(),
INodeDirectory.ROOT_NAME, perms, 0L);
inodes = new INode[]{ rootInode, null };
fs = null;
fsIsReady = true;
}
@Test
public void testNoLimits() throws Exception {
addChildWithName("1", null);
addChildWithName("22", null);
addChildWithName("333", null);
addChildWithName("4444", null);
addChildWithName("55555", null);
addChildWithName(HdfsConstants.DOT_SNAPSHOT_DIR,
mkdirs("/1", null);
mkdirs("/22", null);
mkdirs("/333", null);
mkdirs("/4444", null);
mkdirs("/55555", null);
mkdirs("/1/" + HdfsConstants.DOT_SNAPSHOT_DIR,
HadoopIllegalArgumentException.class);
}
@ -98,33 +103,65 @@ public void testNoLimits() throws Exception {
public void testMaxComponentLength() throws Exception {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_MAX_COMPONENT_LENGTH_KEY, 2);
addChildWithName("1", null);
addChildWithName("22", null);
addChildWithName("333", PathComponentTooLongException.class);
addChildWithName("4444", PathComponentTooLongException.class);
mkdirs("/1", null);
mkdirs("/22", null);
mkdirs("/333", PathComponentTooLongException.class);
mkdirs("/4444", PathComponentTooLongException.class);
}
@Test
public void testMaxComponentLengthRename() throws Exception {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_MAX_COMPONENT_LENGTH_KEY, 2);
mkdirs("/5", null);
rename("/5", "/555", PathComponentTooLongException.class);
rename("/5", "/55", null);
mkdirs("/6", null);
deprecatedRename("/6", "/666", PathComponentTooLongException.class);
deprecatedRename("/6", "/66", null);
}
@Test
public void testMaxDirItems() throws Exception {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_MAX_DIRECTORY_ITEMS_KEY, 2);
addChildWithName("1", null);
addChildWithName("22", null);
addChildWithName("333", MaxDirectoryItemsExceededException.class);
addChildWithName("4444", MaxDirectoryItemsExceededException.class);
mkdirs("/1", null);
mkdirs("/22", null);
mkdirs("/333", MaxDirectoryItemsExceededException.class);
mkdirs("/4444", MaxDirectoryItemsExceededException.class);
}
@Test
public void testMaxDirItemsRename() throws Exception {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_MAX_DIRECTORY_ITEMS_KEY, 2);
mkdirs("/1", null);
mkdirs("/2", null);
mkdirs("/2/A", null);
rename("/2/A", "/A", MaxDirectoryItemsExceededException.class);
rename("/2/A", "/1/A", null);
mkdirs("/2/B", null);
deprecatedRename("/2/B", "/B", MaxDirectoryItemsExceededException.class);
deprecatedRename("/2/B", "/1/B", null);
rename("/1", "/3", null);
deprecatedRename("/2", "/4", null);
}
@Test
public void testMaxDirItemsLimits() throws Exception {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_MAX_DIRECTORY_ITEMS_KEY, 0);
try {
addChildWithName("1", null);
mkdirs("1", null);
} catch (IllegalArgumentException e) {
GenericTestUtils.assertExceptionContains("Cannot set dfs", e);
}
conf.setInt(DFSConfigKeys.DFS_NAMENODE_MAX_DIRECTORY_ITEMS_KEY, 64*100*1024);
try {
addChildWithName("1", null);
mkdirs("1", null);
} catch (IllegalArgumentException e) {
GenericTestUtils.assertExceptionContains("Cannot set dfs", e);
}
@ -135,10 +172,10 @@ public void testMaxComponentsAndMaxDirItems() throws Exception {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_MAX_COMPONENT_LENGTH_KEY, 3);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_MAX_DIRECTORY_ITEMS_KEY, 2);
addChildWithName("1", null);
addChildWithName("22", null);
addChildWithName("333", MaxDirectoryItemsExceededException.class);
addChildWithName("4444", PathComponentTooLongException.class);
mkdirs("/1", null);
mkdirs("/22", null);
mkdirs("/333", MaxDirectoryItemsExceededException.class);
mkdirs("/4444", PathComponentTooLongException.class);
}
@Test
@ -147,32 +184,55 @@ public void testDuringEditLogs() throws Exception {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_MAX_DIRECTORY_ITEMS_KEY, 2);
fsIsReady = false;
addChildWithName(HdfsConstants.DOT_SNAPSHOT_DIR,
mkdirs("/1", null);
mkdirs("/22", null);
mkdirs("/333", null);
mkdirs("/4444", null);
mkdirs("/1/" + HdfsConstants.DOT_SNAPSHOT_DIR,
HadoopIllegalArgumentException.class);
addChildWithName("1", null);
addChildWithName("22", null);
addChildWithName("333", null);
addChildWithName("4444", null);
}
private void addChildWithName(String name, Class<?> expected)
private void mkdirs(String name, Class<?> expected)
throws Exception {
// have to create after the caller has had a chance to set conf values
if (fs == null) fs = new MockFSDirectory();
INode child = new INodeDirectory(getMockNamesystem().allocateNewInodeId(),
DFSUtil.string2Bytes(name), perms, 0L);
lazyInitFSDirectory();
Class<?> generated = null;
try {
fs.verifyMaxComponentLength(child.getLocalNameBytes(), inodes, 1);
fs.verifyMaxDirItems(inodes, 1);
fs.verifyINodeName(child.getLocalNameBytes());
rootInode.addChild(child);
fs.mkdirs(name, perms, false, now());
} catch (Throwable e) {
generated = e.getClass();
}
assertEquals(expected, generated);
}
private void rename(String src, String dst, Class<?> expected)
throws Exception {
lazyInitFSDirectory();
Class<?> generated = null;
try {
fs.renameTo(src, dst, false, new Rename[] { });
} catch (Throwable e) {
generated = e.getClass();
}
assertEquals(expected, generated);
}
@SuppressWarnings("deprecation")
private void deprecatedRename(String src, String dst, Class<?> expected)
throws Exception {
lazyInitFSDirectory();
Class<?> generated = null;
try {
fs.renameTo(src, dst, false);
} catch (Throwable e) {
generated = e.getClass();
}
assertEquals(expected, generated);
}
private static void lazyInitFSDirectory() throws IOException {
// have to create after the caller has had a chance to set conf values
if (fs == null) {
fs = new MockFSDirectory();
}
}
}