diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 359a38bdd7..2f8acb0c79 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -566,6 +566,9 @@ Release 2.8.0 - UNRELEASED HADOOP-9723. Improve error message when hadoop archive output path already exists. (Jean-Baptiste Onofré and Yongjun Zhang via aajisak) + HADOOP-11713. ViewFileSystem should support snapshot methods. + (Rakesh R via cnauroth) + OPTIMIZATIONS HADOOP-11785. Reduce the number of listStatus operation in distcp diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java index 18e2391ee2..f7a93e7818 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java @@ -363,6 +363,23 @@ public void removeXAttr(Path path, String name) throws IOException { super.removeXAttr(fullPath(path), name); } + @Override + public Path createSnapshot(Path path, String name) throws IOException { + return super.createSnapshot(fullPath(path), name); + } + + @Override + public void renameSnapshot(Path path, String snapshotOldName, + String snapshotNewName) throws IOException { + super.renameSnapshot(fullPath(path), snapshotOldName, snapshotNewName); + } + + @Override + public void deleteSnapshot(Path snapshotDir, String snapshotName) + throws IOException { + super.deleteSnapshot(fullPath(snapshotDir), snapshotName); + } + @Override public Path resolvePath(final Path p) throws IOException { return super.resolvePath(fullPath(p)); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java index 68e756a81b..a05a7000f5 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java @@ -361,7 +361,24 @@ public void removeXAttr(Path path, String name) throws IOException { } @Override - public void setVerifyChecksum(final boolean verifyChecksum) + public Path createSnapshot(Path path, String name) throws IOException { + return myFs.createSnapshot(fullPath(path), name); + } + + @Override + public void renameSnapshot(Path path, String snapshotOldName, + String snapshotNewName) throws IOException { + myFs.renameSnapshot(fullPath(path), snapshotOldName, snapshotNewName); + } + + @Override + public void deleteSnapshot(Path snapshotDir, String snapshotName) + throws IOException { + myFs.deleteSnapshot(fullPath(snapshotDir), snapshotName); + } + + @Override + public void setVerifyChecksum(final boolean verifyChecksum) throws IOException, UnresolvedLinkException { myFs.setVerifyChecksum(verifyChecksum); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index 43fe23fb82..860506401b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -726,7 +726,32 @@ public MountPoint[] getMountPoints() { } return result; } - + + @Override + public Path createSnapshot(Path path, String snapshotName) + throws IOException { + InodeTree.ResolveResult res = fsState.resolve(getUriPath(path), + true); + return res.targetFileSystem.createSnapshot(res.remainingPath, snapshotName); + } + + @Override + public void renameSnapshot(Path path, String snapshotOldName, + String snapshotNewName) throws IOException { + InodeTree.ResolveResult res = fsState.resolve(getUriPath(path), + true); + res.targetFileSystem.renameSnapshot(res.remainingPath, snapshotOldName, + snapshotNewName); + } + + @Override + public void deleteSnapshot(Path path, String snapshotName) + throws IOException { + InodeTree.ResolveResult res = fsState.resolve(getUriPath(path), + true); + res.targetFileSystem.deleteSnapshot(res.remainingPath, snapshotName); + } + /* * An instance of this class represents an internal dir of the viewFs * that is internal dir of the mount table. @@ -1020,5 +1045,26 @@ public void removeXAttr(Path path, String name) throws IOException { checkPathIsSlash(path); throw readOnlyMountTable("removeXAttr", path); } + + @Override + public Path createSnapshot(Path path, String snapshotName) + throws IOException { + checkPathIsSlash(path); + throw readOnlyMountTable("createSnapshot", path); + } + + @Override + public void renameSnapshot(Path path, String snapshotOldName, + String snapshotNewName) throws IOException { + checkPathIsSlash(path); + throw readOnlyMountTable("renameSnapshot", path); + } + + @Override + public void deleteSnapshot(Path path, String snapshotName) + throws IOException { + checkPathIsSlash(path); + throw readOnlyMountTable("deleteSnapshot", path); + } } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java index 975496aa42..a23aa869ee 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java @@ -621,7 +621,8 @@ public List> getDelegationTokens(String renewer) throws IOException { @Override public boolean isValidName(String src) { - // Prefix validated at mount time and rest of path validated by mount target. + // Prefix validated at mount time and rest of path validated by mount + // target. return true; } @@ -714,8 +715,31 @@ public void removeXAttr(Path path, String name) throws IOException { fsState.resolve(getUriPath(path), true); res.targetFileSystem.removeXAttr(res.remainingPath, name); } - - + + @Override + public Path createSnapshot(Path path, String snapshotName) + throws IOException { + InodeTree.ResolveResult res = fsState.resolve( + getUriPath(path), true); + return res.targetFileSystem.createSnapshot(res.remainingPath, snapshotName); + } + + @Override + public void renameSnapshot(Path path, String snapshotOldName, + String snapshotNewName) throws IOException { + InodeTree.ResolveResult res = fsState.resolve( + getUriPath(path), true); + res.targetFileSystem.renameSnapshot(res.remainingPath, snapshotOldName, + snapshotNewName); + } + + @Override + public void deleteSnapshot(Path path, String snapshotName) throws IOException { + InodeTree.ResolveResult res = fsState.resolve( + getUriPath(path), true); + res.targetFileSystem.deleteSnapshot(res.remainingPath, snapshotName); + } + /* * An instance of this class represents an internal dir of the viewFs * ie internal dir of the mount table. @@ -1025,5 +1049,26 @@ public void removeXAttr(Path path, String name) throws IOException { checkPathIsSlash(path); throw readOnlyMountTable("removeXAttr", path); } + + @Override + public Path createSnapshot(Path path, String snapshotName) + throws IOException { + checkPathIsSlash(path); + throw readOnlyMountTable("createSnapshot", path); + } + + @Override + public void renameSnapshot(Path path, String snapshotOldName, + String snapshotNewName) throws IOException { + checkPathIsSlash(path); + throw readOnlyMountTable("renameSnapshot", path); + } + + @Override + public void deleteSnapshot(Path path, String snapshotName) + throws IOException { + checkPathIsSlash(path); + throw readOnlyMountTable("deleteSnapshot", path); + } } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java index a13ee8d070..e4a1ac9729 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java @@ -416,4 +416,56 @@ static class MockFileSystem extends FilterFileSystem { @Override public void initialize(URI name, Configuration conf) throws IOException {} } + + @Test(timeout = 30000) + public void testCreateSnapshot() throws Exception { + Path snapRootPath = new Path("/snapPath"); + Path chRootedSnapRootPath = new Path("/a/b/snapPath"); + + Configuration conf = new Configuration(); + conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); + + URI chrootUri = URI.create("mockfs://foo/a/b"); + ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf); + FileSystem mockFs = ((FilterFileSystem) chrootFs.getRawFileSystem()) + .getRawFileSystem(); + + chrootFs.createSnapshot(snapRootPath, "snap1"); + verify(mockFs).createSnapshot(chRootedSnapRootPath, "snap1"); + } + + @Test(timeout = 30000) + public void testDeleteSnapshot() throws Exception { + Path snapRootPath = new Path("/snapPath"); + Path chRootedSnapRootPath = new Path("/a/b/snapPath"); + + Configuration conf = new Configuration(); + conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); + + URI chrootUri = URI.create("mockfs://foo/a/b"); + ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf); + FileSystem mockFs = ((FilterFileSystem) chrootFs.getRawFileSystem()) + .getRawFileSystem(); + + chrootFs.deleteSnapshot(snapRootPath, "snap1"); + verify(mockFs).deleteSnapshot(chRootedSnapRootPath, "snap1"); + } + + @Test(timeout = 30000) + public void testRenameSnapshot() throws Exception { + Path snapRootPath = new Path("/snapPath"); + Path chRootedSnapRootPath = new Path("/a/b/snapPath"); + + Configuration conf = new Configuration(); + conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); + + URI chrootUri = URI.create("mockfs://foo/a/b"); + ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf); + FileSystem mockFs = ((FilterFileSystem) chrootFs.getRawFileSystem()) + .getRawFileSystem(); + + chrootFs.renameSnapshot(snapRootPath, "snapOldName", "snapNewName"); + verify(mockFs).renameSnapshot(chRootedSnapRootPath, "snapOldName", + "snapNewName"); + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java index e639291c65..20825e312c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java @@ -327,4 +327,45 @@ public void testIsValidNameInvalidInBaseFs() throws Exception { Assert.assertFalse(chRootedFs.isValidName("/test")); Mockito.verify(baseFs).isValidName("/chroot/test"); } + + @Test(timeout = 30000) + public void testCreateSnapshot() throws Exception { + Path snapRootPath = new Path("/snapPath"); + Path chRootedSnapRootPath = new Path( + Path.getPathWithoutSchemeAndAuthority(chrootedTo), "snapPath"); + AbstractFileSystem baseFs = Mockito.spy(fc.getDefaultFileSystem()); + ChRootedFs chRootedFs = new ChRootedFs(baseFs, chrootedTo); + Mockito.doReturn(snapRootPath).when(baseFs) + .createSnapshot(chRootedSnapRootPath, "snap1"); + Assert.assertEquals(snapRootPath, + chRootedFs.createSnapshot(snapRootPath, "snap1")); + Mockito.verify(baseFs).createSnapshot(chRootedSnapRootPath, "snap1"); + } + + @Test(timeout = 30000) + public void testDeleteSnapshot() throws Exception { + Path snapRootPath = new Path("/snapPath"); + Path chRootedSnapRootPath = new Path( + Path.getPathWithoutSchemeAndAuthority(chrootedTo), "snapPath"); + AbstractFileSystem baseFs = Mockito.spy(fc.getDefaultFileSystem()); + ChRootedFs chRootedFs = new ChRootedFs(baseFs, chrootedTo); + Mockito.doNothing().when(baseFs) + .deleteSnapshot(chRootedSnapRootPath, "snap1"); + chRootedFs.deleteSnapshot(snapRootPath, "snap1"); + Mockito.verify(baseFs).deleteSnapshot(chRootedSnapRootPath, "snap1"); + } + + @Test(timeout = 30000) + public void testRenameSnapshot() throws Exception { + Path snapRootPath = new Path("/snapPath"); + Path chRootedSnapRootPath = new Path( + Path.getPathWithoutSchemeAndAuthority(chrootedTo), "snapPath"); + AbstractFileSystem baseFs = Mockito.spy(fc.getDefaultFileSystem()); + ChRootedFs chRootedFs = new ChRootedFs(baseFs, chrootedTo); + Mockito.doNothing().when(baseFs) + .renameSnapshot(chRootedSnapRootPath, "snapOldName", "snapNewName"); + chRootedFs.renameSnapshot(snapRootPath, "snapOldName", "snapNewName"); + Mockito.verify(baseFs).renameSnapshot(chRootedSnapRootPath, "snapOldName", + "snapNewName"); + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java index 18769c278f..7fad990fb8 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java @@ -865,4 +865,24 @@ public void testInternalRemoveXAttr() throws IOException { fsView.removeXAttr(new Path("/internalDir"), "xattrName"); } + @Test(expected = AccessControlException.class) + public void testInternalCreateSnapshot1() throws IOException { + fsView.createSnapshot(new Path("/internalDir")); + } + + @Test(expected = AccessControlException.class) + public void testInternalCreateSnapshot2() throws IOException { + fsView.createSnapshot(new Path("/internalDir"), "snap1"); + } + + @Test(expected = AccessControlException.class) + public void testInternalRenameSnapshot() throws IOException { + fsView.renameSnapshot(new Path("/internalDir"), "snapOldName", + "snapNewName"); + } + + @Test(expected = AccessControlException.class) + public void testInternalDeleteSnapshot() throws IOException { + fsView.deleteSnapshot(new Path("/internalDir"), "snap1"); + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java index 035b280249..d8ab53911c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java @@ -777,4 +777,25 @@ public void testInternalListXAttr() throws IOException { public void testInternalRemoveXAttr() throws IOException { fcView.removeXAttr(new Path("/internalDir"), "xattrName"); } + + @Test(expected = AccessControlException.class) + public void testInternalCreateSnapshot1() throws IOException { + fcView.createSnapshot(new Path("/internalDir")); + } + + @Test(expected = AccessControlException.class) + public void testInternalCreateSnapshot2() throws IOException { + fcView.createSnapshot(new Path("/internalDir"), "snap1"); + } + + @Test(expected = AccessControlException.class) + public void testInternalRenameSnapshot() throws IOException { + fcView.renameSnapshot(new Path("/internalDir"), "snapOldName", + "snapNewName"); + } + + @Test(expected = AccessControlException.class) + public void testInternalDeleteSnapshot() throws IOException { + fcView.deleteSnapshot(new Path("/internalDir"), "snap1"); + } }