big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-11713. ViewFileSystem should support snapshot methods. Contributed by Rakesh R.

Browse Source
This commit is contained in:
cnauroth 2015-05-14 14:55:58 -07:00
parent 15ccd967ee
commit 09fe16f166
9 changed files with 267 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -566,6 +566,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-9723. Improve error message when hadoop archive output path already
exists. (Jean-Baptiste Onofré and Yongjun Zhang via aajisak)
HADOOP-11713. ViewFileSystem should support snapshot methods.
(Rakesh R via cnauroth)
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp

17
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
View File

@ -363,6 +363,23 @@ public void removeXAttr(Path path, String name) throws IOException {
super.removeXAttr(fullPath(path), name);
}
@Override
public Path createSnapshot(Path path, String name) throws IOException {
return super.createSnapshot(fullPath(path), name);
}
@Override
public void renameSnapshot(Path path, String snapshotOldName,
String snapshotNewName) throws IOException {
super.renameSnapshot(fullPath(path), snapshotOldName, snapshotNewName);
}
@Override
public void deleteSnapshot(Path snapshotDir, String snapshotName)
throws IOException {
super.deleteSnapshot(fullPath(snapshotDir), snapshotName);
}
@Override
public Path resolvePath(final Path p) throws IOException {
return super.resolvePath(fullPath(p));

19
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
View File

@ -361,7 +361,24 @@ public void removeXAttr(Path path, String name) throws IOException {
}
@Override
public void setVerifyChecksum(final boolean verifyChecksum)
public Path createSnapshot(Path path, String name) throws IOException {
return myFs.createSnapshot(fullPath(path), name);
}
@Override
public void renameSnapshot(Path path, String snapshotOldName,
String snapshotNewName) throws IOException {
myFs.renameSnapshot(fullPath(path), snapshotOldName, snapshotNewName);
}
@Override
public void deleteSnapshot(Path snapshotDir, String snapshotName)
throws IOException {
myFs.deleteSnapshot(fullPath(snapshotDir), snapshotName);
}
@Override
public void setVerifyChecksum(final boolean verifyChecksum)
throws IOException, UnresolvedLinkException {
myFs.setVerifyChecksum(verifyChecksum);
}

48
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
View File

@ -726,7 +726,32 @@ public MountPoint[] getMountPoints() {
}
return result;
}
@Override
public Path createSnapshot(Path path, String snapshotName)
throws IOException {
InodeTree.ResolveResult<FileSystem> res = fsState.resolve(getUriPath(path),
true);
return res.targetFileSystem.createSnapshot(res.remainingPath, snapshotName);
}
@Override
public void renameSnapshot(Path path, String snapshotOldName,
String snapshotNewName) throws IOException {
InodeTree.ResolveResult<FileSystem> res = fsState.resolve(getUriPath(path),
true);
res.targetFileSystem.renameSnapshot(res.remainingPath, snapshotOldName,
snapshotNewName);
}
@Override
public void deleteSnapshot(Path path, String snapshotName)
throws IOException {
InodeTree.ResolveResult<FileSystem> res = fsState.resolve(getUriPath(path),
true);
res.targetFileSystem.deleteSnapshot(res.remainingPath, snapshotName);
}
/*
* An instance of this class represents an internal dir of the viewFs
* that is internal dir of the mount table.
@ -1020,5 +1045,26 @@ public void removeXAttr(Path path, String name) throws IOException {
checkPathIsSlash(path);
throw readOnlyMountTable("removeXAttr", path);
}
@Override
public Path createSnapshot(Path path, String snapshotName)
throws IOException {
checkPathIsSlash(path);
throw readOnlyMountTable("createSnapshot", path);
}
@Override
public void renameSnapshot(Path path, String snapshotOldName,
String snapshotNewName) throws IOException {
checkPathIsSlash(path);
throw readOnlyMountTable("renameSnapshot", path);
}
@Override
public void deleteSnapshot(Path path, String snapshotName)
throws IOException {
checkPathIsSlash(path);
throw readOnlyMountTable("deleteSnapshot", path);
}
}
}

51
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
View File

@ -621,7 +621,8 @@ public List<Token<?>> getDelegationTokens(String renewer) throws IOException {
@Override
public boolean isValidName(String src) {
// Prefix validated at mount time and rest of path validated by mount target.
// Prefix validated at mount time and rest of path validated by mount
// target.
return true;
}
@ -714,8 +715,31 @@ public void removeXAttr(Path path, String name) throws IOException {
fsState.resolve(getUriPath(path), true);
res.targetFileSystem.removeXAttr(res.remainingPath, name);
}
@Override
public Path createSnapshot(Path path, String snapshotName)
throws IOException {
InodeTree.ResolveResult<AbstractFileSystem> res = fsState.resolve(
getUriPath(path), true);
return res.targetFileSystem.createSnapshot(res.remainingPath, snapshotName);
}
@Override
public void renameSnapshot(Path path, String snapshotOldName,
String snapshotNewName) throws IOException {
InodeTree.ResolveResult<AbstractFileSystem> res = fsState.resolve(
getUriPath(path), true);
res.targetFileSystem.renameSnapshot(res.remainingPath, snapshotOldName,
snapshotNewName);
}
@Override
public void deleteSnapshot(Path path, String snapshotName) throws IOException {
InodeTree.ResolveResult<AbstractFileSystem> res = fsState.resolve(
getUriPath(path), true);
res.targetFileSystem.deleteSnapshot(res.remainingPath, snapshotName);
}
/*
* An instance of this class represents an internal dir of the viewFs
* ie internal dir of the mount table.
@ -1025,5 +1049,26 @@ public void removeXAttr(Path path, String name) throws IOException {
checkPathIsSlash(path);
throw readOnlyMountTable("removeXAttr", path);
}
@Override
public Path createSnapshot(Path path, String snapshotName)
throws IOException {
checkPathIsSlash(path);
throw readOnlyMountTable("createSnapshot", path);
}
@Override
public void renameSnapshot(Path path, String snapshotOldName,
String snapshotNewName) throws IOException {
checkPathIsSlash(path);
throw readOnlyMountTable("renameSnapshot", path);
}
@Override
public void deleteSnapshot(Path path, String snapshotName)
throws IOException {
checkPathIsSlash(path);
throw readOnlyMountTable("deleteSnapshot", path);
}
}
}

52
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java
View File

@ -416,4 +416,56 @@ static class MockFileSystem extends FilterFileSystem {
@Override
public void initialize(URI name, Configuration conf) throws IOException {}
}
@Test(timeout = 30000)
public void testCreateSnapshot() throws Exception {
Path snapRootPath = new Path("/snapPath");
Path chRootedSnapRootPath = new Path("/a/b/snapPath");
Configuration conf = new Configuration();
conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
URI chrootUri = URI.create("mockfs://foo/a/b");
ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf);
FileSystem mockFs = ((FilterFileSystem) chrootFs.getRawFileSystem())
.getRawFileSystem();
chrootFs.createSnapshot(snapRootPath, "snap1");
verify(mockFs).createSnapshot(chRootedSnapRootPath, "snap1");
}
@Test(timeout = 30000)
public void testDeleteSnapshot() throws Exception {
Path snapRootPath = new Path("/snapPath");
Path chRootedSnapRootPath = new Path("/a/b/snapPath");
Configuration conf = new Configuration();
conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
URI chrootUri = URI.create("mockfs://foo/a/b");
ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf);
FileSystem mockFs = ((FilterFileSystem) chrootFs.getRawFileSystem())
.getRawFileSystem();
chrootFs.deleteSnapshot(snapRootPath, "snap1");
verify(mockFs).deleteSnapshot(chRootedSnapRootPath, "snap1");
}
@Test(timeout = 30000)
public void testRenameSnapshot() throws Exception {
Path snapRootPath = new Path("/snapPath");
Path chRootedSnapRootPath = new Path("/a/b/snapPath");
Configuration conf = new Configuration();
conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
URI chrootUri = URI.create("mockfs://foo/a/b");
ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf);
FileSystem mockFs = ((FilterFileSystem) chrootFs.getRawFileSystem())
.getRawFileSystem();
chrootFs.renameSnapshot(snapRootPath, "snapOldName", "snapNewName");
verify(mockFs).renameSnapshot(chRootedSnapRootPath, "snapOldName",
"snapNewName");
}
}

41
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFs.java
View File

@ -327,4 +327,45 @@ public void testIsValidNameInvalidInBaseFs() throws Exception {
Assert.assertFalse(chRootedFs.isValidName("/test"));
Mockito.verify(baseFs).isValidName("/chroot/test");
}
@Test(timeout = 30000)
public void testCreateSnapshot() throws Exception {
Path snapRootPath = new Path("/snapPath");
Path chRootedSnapRootPath = new Path(
Path.getPathWithoutSchemeAndAuthority(chrootedTo), "snapPath");
AbstractFileSystem baseFs = Mockito.spy(fc.getDefaultFileSystem());
ChRootedFs chRootedFs = new ChRootedFs(baseFs, chrootedTo);
Mockito.doReturn(snapRootPath).when(baseFs)
.createSnapshot(chRootedSnapRootPath, "snap1");
Assert.assertEquals(snapRootPath,
chRootedFs.createSnapshot(snapRootPath, "snap1"));
Mockito.verify(baseFs).createSnapshot(chRootedSnapRootPath, "snap1");
}
@Test(timeout = 30000)
public void testDeleteSnapshot() throws Exception {
Path snapRootPath = new Path("/snapPath");
Path chRootedSnapRootPath = new Path(
Path.getPathWithoutSchemeAndAuthority(chrootedTo), "snapPath");
AbstractFileSystem baseFs = Mockito.spy(fc.getDefaultFileSystem());
ChRootedFs chRootedFs = new ChRootedFs(baseFs, chrootedTo);
Mockito.doNothing().when(baseFs)
.deleteSnapshot(chRootedSnapRootPath, "snap1");
chRootedFs.deleteSnapshot(snapRootPath, "snap1");
Mockito.verify(baseFs).deleteSnapshot(chRootedSnapRootPath, "snap1");
}
@Test(timeout = 30000)
public void testRenameSnapshot() throws Exception {
Path snapRootPath = new Path("/snapPath");
Path chRootedSnapRootPath = new Path(
Path.getPathWithoutSchemeAndAuthority(chrootedTo), "snapPath");
AbstractFileSystem baseFs = Mockito.spy(fc.getDefaultFileSystem());
ChRootedFs chRootedFs = new ChRootedFs(baseFs, chrootedTo);
Mockito.doNothing().when(baseFs)
.renameSnapshot(chRootedSnapRootPath, "snapOldName", "snapNewName");
chRootedFs.renameSnapshot(snapRootPath, "snapOldName", "snapNewName");
Mockito.verify(baseFs).renameSnapshot(chRootedSnapRootPath, "snapOldName",
"snapNewName");
}
}

20
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
View File

@ -865,4 +865,24 @@ public void testInternalRemoveXAttr() throws IOException {
fsView.removeXAttr(new Path("/internalDir"), "xattrName");
}
@Test(expected = AccessControlException.class)
public void testInternalCreateSnapshot1() throws IOException {
fsView.createSnapshot(new Path("/internalDir"));
}
@Test(expected = AccessControlException.class)
public void testInternalCreateSnapshot2() throws IOException {
fsView.createSnapshot(new Path("/internalDir"), "snap1");
}
@Test(expected = AccessControlException.class)
public void testInternalRenameSnapshot() throws IOException {
fsView.renameSnapshot(new Path("/internalDir"), "snapOldName",
"snapNewName");
}
@Test(expected = AccessControlException.class)
public void testInternalDeleteSnapshot() throws IOException {
fsView.deleteSnapshot(new Path("/internalDir"), "snap1");
}
}

21
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
View File

@ -777,4 +777,25 @@ public void testInternalListXAttr() throws IOException {
public void testInternalRemoveXAttr() throws IOException {
fcView.removeXAttr(new Path("/internalDir"), "xattrName");
}
@Test(expected = AccessControlException.class)
public void testInternalCreateSnapshot1() throws IOException {
fcView.createSnapshot(new Path("/internalDir"));
}
@Test(expected = AccessControlException.class)
public void testInternalCreateSnapshot2() throws IOException {
fcView.createSnapshot(new Path("/internalDir"), "snap1");
}
@Test(expected = AccessControlException.class)
public void testInternalRenameSnapshot() throws IOException {
fcView.renameSnapshot(new Path("/internalDir"), "snapOldName",
"snapNewName");
}
@Test(expected = AccessControlException.class)
public void testInternalDeleteSnapshot() throws IOException {
fcView.deleteSnapshot(new Path("/internalDir"), "snap1");
}
}