big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-15446. CreateSnapshotOp fails during edit log loading for /.reserved/raw/path with error java.io.FileNotFoundException: Directory does not exist: /.reserved/raw/path. Contributed by Stephen O'Donnell.

Browse Source
This commit is contained in:
Ayush Saxena 2020-07-04 12:24:49 +05:30
parent 1f2a80b5e5
commit f86f15cf20
3 changed files with 76 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
View File

@ -734,6 +734,26 @@ public INodesInPath resolvePath(FSPermissionChecker pc, String src,
return iip;
}
/**
* This method should only be used from internal paths and not those provided
* directly by a user. It resolves a given path into an INodesInPath in a
* similar way to resolvePath(...), only traversal and permissions are not
* checked.
* @param src The path to resolve.
* @return if the path indicates an inode, return path after replacing up to
* {@code <inodeid>} with the corresponding path of the inode, else
* the path in {@code src} as is. If the path refers to a path in
* the "raw" directory, return the non-raw pathname.
* @throws FileNotFoundException
*/
public INodesInPath unprotectedResolvePath(String src)
throws FileNotFoundException {
byte[][] components = INode.getPathComponents(src);
boolean isRaw = isReservedRawName(components);
components = resolveComponents(components, this);
return INodesInPath.resolve(rootDir, components, isRaw);
}
INodesInPath resolvePath(FSPermissionChecker pc, String src, long fileId)
throws UnresolvedLinkException, FileNotFoundException,
AccessControlException, ParentNotDirectoryException {

6
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
View File

@ -798,7 +798,7 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
final String snapshotRoot =
renameReservedPathsOnUpgrade(createSnapshotOp.snapshotRoot,
logVersion);
INodesInPath iip = fsDir.getINodesInPath(snapshotRoot, DirOp.WRITE);
INodesInPath iip = fsDir.unprotectedResolvePath(snapshotRoot);
String path = fsNamesys.getSnapshotManager().createSnapshot(
fsDir.getFSNamesystem().getLeaseManager(),
iip, snapshotRoot, createSnapshotOp.snapshotName,
@ -816,7 +816,7 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
final String snapshotRoot =
renameReservedPathsOnUpgrade(deleteSnapshotOp.snapshotRoot,
logVersion);
INodesInPath iip = fsDir.getINodesInPath(snapshotRoot, DirOp.WRITE);
INodesInPath iip = fsDir.unprotectedResolvePath(snapshotRoot);
fsNamesys.getSnapshotManager().deleteSnapshot(iip,
deleteSnapshotOp.snapshotName,
new INode.ReclaimContext(fsNamesys.dir.getBlockStoragePolicySuite(),
@ -838,7 +838,7 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
final String snapshotRoot =
renameReservedPathsOnUpgrade(renameSnapshotOp.snapshotRoot,
logVersion);
INodesInPath iip = fsDir.getINodesInPath(snapshotRoot, DirOp.WRITE);
INodesInPath iip = fsDir.unprotectedResolvePath(snapshotRoot);
fsNamesys.getSnapshotManager().renameSnapshot(iip,
snapshotRoot, renameSnapshotOp.snapshotOldName,
renameSnapshotOp.snapshotNewName, renameSnapshotOp.mtime);

53
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshot.java
View File

@ -513,6 +513,59 @@ public void testDeletionSnapshotMtime() throws Exception {
newSnapshotStatus.getModificationTime());
}
/**
* HDFS-15446 - ensure that snapshot operations on /.reserved/raw
* paths work and the NN can load the resulting edits.
*/
@Test(timeout = 60000)
public void testSnapshotOpsOnReservedPath() throws Exception {
Path dir = new Path("/dir");
Path nestedDir = new Path("/nested/dir");
Path sub = new Path(dir, "sub");
Path subFile = new Path(sub, "file");
Path nestedFile = new Path(nestedDir, "file");
DFSTestUtil.createFile(hdfs, subFile, BLOCKSIZE, REPLICATION, seed);
DFSTestUtil.createFile(hdfs, nestedFile, BLOCKSIZE, REPLICATION, seed);
hdfs.allowSnapshot(dir);
hdfs.allowSnapshot(nestedDir);
Path reservedDir = new Path("/.reserved/raw/dir");
Path reservedNestedDir = new Path("/.reserved/raw/nested/dir");
hdfs.createSnapshot(reservedDir, "s1");
hdfs.createSnapshot(reservedNestedDir, "s1");
hdfs.renameSnapshot(reservedDir, "s1", "s2");
hdfs.renameSnapshot(reservedNestedDir, "s1", "s2");
hdfs.deleteSnapshot(reservedDir, "s2");
hdfs.deleteSnapshot(reservedNestedDir, "s2");
// The original problem with reserved path, is that the NN was unable to
// replay the edits, therefore restarting the NN to ensure it starts
// and no exceptions are raised.
cluster.restartNameNode(true);
}
/**
* HDFS-15446 - ensure that snapshot operations on /.reserved/raw
* paths work and the NN can load the resulting edits. This test if for
* snapshots at the root level.
*/
@Test(timeout = 60000)
public void testSnapshotOpsOnRootReservedPath() throws Exception {
Path dir = new Path("/");
Path sub = new Path(dir, "sub");
Path subFile = new Path(sub, "file");
DFSTestUtil.createFile(hdfs, subFile, BLOCKSIZE, REPLICATION, seed);
hdfs.allowSnapshot(dir);
Path reservedDir = new Path("/.reserved/raw");
hdfs.createSnapshot(reservedDir, "s1");
hdfs.renameSnapshot(reservedDir, "s1", "s2");
hdfs.deleteSnapshot(reservedDir, "s2");
// The original problem with reserved path, is that the NN was unable to
// replay the edits, therefore restarting the NN to ensure it starts
// and no exceptions are raised.
cluster.restartNameNode(true);
}
/**
* Prepare a list of modifications. A modification may be a file creation,
* file deletion, or a modification operation such as appending to an existing