big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-18110. ViewFileSystem: Add Support for Localized Trash Root

Browse Source 
Fixes #3956
This commit is contained in:
Xing Lin 2022-01-26 14:48:59 -08:00 committed by Owen O'Malley
parent fe583c4b63
commit ca8ba24051
3 changed files with 255 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
View File

@ -132,4 +132,10 @@ public interface Constants {
Class<? extends MountTableConfigLoader>
DEFAULT_MOUNT_TABLE_CONFIG_LOADER_IMPL =
HCFSMountTableConfigLoader.class;
/**
* Enable ViewFileSystem to return a trashRoot which is local to mount point.
*/
String CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH = "fs.viewfs.mount.point.local.trash";
boolean CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH_DEFAULT = false;
}

95
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
View File

@ -25,6 +25,8 @@
import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS;
import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS_DEFAULT;
import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555;
import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH;
import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH_DEFAULT;
import java.util.function.Function;
import java.io.FileNotFoundException;
@ -1130,16 +1132,49 @@ public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
* Get the trash root directory for current user when the path
* specified is deleted.
*
* If CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH is not set, return
* the default trash root from targetFS.
*
* When CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH is set to true,
* 1) If path p is in fallback FS or from the same mount point as the default
* trash root for targetFS, return the default trash root for targetFS.
* 2) else, return a trash root in the mounted targetFS
* (/mntpoint/.Trash/{user})
*
* @param path the trash root of the path to be determined.
* @return the trash root path.
*/
@Override
public Path getTrashRoot(Path path) {
boolean useMountPointLocalTrash =
config.getBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH,
CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH_DEFAULT);
try {
InodeTree.ResolveResult<FileSystem> res =
fsState.resolve(getUriPath(path), true);
return res.targetFileSystem.getTrashRoot(res.remainingPath);
} catch (Exception e) {
Path trashRoot = res.targetFileSystem.getTrashRoot(res.remainingPath);
if (!useMountPointLocalTrash) {
return trashRoot;
} else {
// Path p is either in a mount point or in the fallback FS
if (ROOT_PATH.equals(new Path(res.resolvedPath))
|| trashRoot.toUri().getPath().startsWith(res.resolvedPath)) {
// Path p is in the fallback FS or targetFileSystem.trashRoot is in
// the same mount point as Path p
return trashRoot;
} else {
// targetFileSystem.trashRoot is in a different mount point from
// Path p. Return the trash root for the mount point.
Path mountPointRoot =
res.targetFileSystem.getFileStatus(new Path("/")).getPath();
return new Path(mountPointRoot,
TRASH_PREFIX + "/" + ugi.getShortUserName());
}
}
} catch (IOException | IllegalArgumentException e) {
throw new NotInMountpointException(path, "getTrashRoot");
}
}
@ -1156,6 +1191,62 @@ public Collection<FileStatus> getTrashRoots(boolean allUsers) {
for (FileSystem fs : getChildFileSystems()) {
trashRoots.addAll(fs.getTrashRoots(allUsers));
}
// Add trash dirs for each mount point
boolean useMountPointLocalTrash =
config.getBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH,
CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH_DEFAULT);
if (useMountPointLocalTrash) {
Set<Path> currentTrashPaths = new HashSet<>();
for (FileStatus file : trashRoots) {
currentTrashPaths.add(file.getPath());
}
MountPoint[] mountPoints = getMountPoints();
try {
for (int i = 0; i < mountPoints.length; i++) {
Path trashRoot = makeQualified(
new Path(mountPoints[i].mountedOnPath + "/" + TRASH_PREFIX));
// Continue if trashRoot does not exist for this filesystem
if (!exists(trashRoot)) {
continue;
}
InodeTree.ResolveResult<FileSystem> res =
fsState.resolve(getUriPath(trashRoot), true);
if (!allUsers) {
Path userTrash =
new Path("/" + TRASH_PREFIX + "/" + ugi.getShortUserName());
try {
FileStatus file = res.targetFileSystem.getFileStatus(userTrash);
if (!currentTrashPaths.contains(file.getPath())) {
trashRoots.add(file);
currentTrashPaths.add(file.getPath());
}
} catch (FileNotFoundException ignored) {
}
} else {
FileStatus[] targetFsTrashRoots =
res.targetFileSystem.listStatus(new Path("/" + TRASH_PREFIX));
for (FileStatus file : targetFsTrashRoots) {
// skip if we already include it in currentTrashPaths
if (currentTrashPaths.contains(file.getPath())) {
continue;
}
trashRoots.add(file);
currentTrashPaths.add(file.getPath());
}
}
}
} catch (IOException e) {
LOG.warn("Exception in get all trash roots", e);
}
}
return trashRoots;
}

156
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
View File

@ -69,6 +69,8 @@
import static org.apache.hadoop.fs.FileSystemTestHelper.*;
import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE;
import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555;
import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH;
import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX;
import org.junit.After;
import org.junit.Assert;
@ -1102,6 +1104,160 @@ public void testTrashRoot() throws IOException {
Assert.assertTrue("", fsView.getTrashRoots(true).size() > 0);
}
/**
* Test the localized trash root for getTrashRoot.
*/
@Test
public void testTrashRootLocalizedTrash() throws IOException {
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
Configuration conf2 = new Configuration(conf);
conf2.setBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, true);
ConfigUtil.addLinkFallback(conf2, targetTestRoot.toUri());
FileSystem fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2);
// Case 1: path p not in the default FS.
// Return a trash root within the mount point.
Path dataTestPath = new Path("/data/dir/file");
Path dataTrashRoot = new Path(targetTestRoot,
"data/" + TRASH_PREFIX + "/" + ugi.getShortUserName());
Assert.assertEquals(dataTrashRoot, fsView2.getTrashRoot(dataTestPath));
// Case 2: path p not found in mount table, fall back to the default FS
// Return a trash root in user home dir
Path nonExistentPath = new Path("/nonExistentDir/nonExistentFile");
Path userTrashRoot = new Path(fsTarget.getHomeDirectory(), TRASH_PREFIX);
Assert.assertEquals(userTrashRoot, fsView2.getTrashRoot(nonExistentPath));
// Case 3: turn off the CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH flag.
// Return a trash root in user home dir.
conf2.setBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, false);
fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2);
Assert.assertEquals(userTrashRoot, fsView2.getTrashRoot(dataTestPath));
// Case 4: viewFS without fallback. Expect exception for a nonExistent path
conf2 = new Configuration(conf);
fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2);
try {
fsView2.getTrashRoot(nonExistentPath);
} catch (NotInMountpointException ignored) {
}
// Case 5: path p is in the same mount point as targetFS.getTrashRoot().
// Return targetFS.getTrashRoot()
// Use a new Configuration object, so that we can start with an empty
// mount table. This would avoid a conflict between the /user link in
// setupMountPoints() and homeDir we will need to setup for this test.
// default homeDir for hdfs is /user/.
Configuration conf3 = ViewFileSystemTestSetup.createConfig();
Path homeDir = fsTarget.getHomeDirectory();
String homeParentDir = homeDir.getParent().toUri().getPath();
conf3.setBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, true);
ConfigUtil.addLink(conf3, homeParentDir,
new Path(targetTestRoot, homeParentDir).toUri());
Path homeTestPath = new Path(homeDir.toUri().getPath(), "testuser/file");
FileSystem fsView3 = FileSystem.get(FsConstants.VIEWFS_URI, conf3);
Assert.assertEquals(userTrashRoot, fsView3.getTrashRoot(homeTestPath));
}
/**
* A mocked FileSystem which returns a deep trash dir.
*/
static class MockTrashRootFS extends MockFileSystem {
public static final Path TRASH =
new Path("/mnt/very/deep/deep/trash/dir/.Trash");
@Override
public Path getTrashRoot(Path path) {
return TRASH;
}
}
/**
* Test a trash root that is inside a mount point for getTrashRoot
*/
@Test
public void testTrashRootDeepTrashDir() throws IOException {
Configuration conf2 = ViewFileSystemTestSetup.createConfig();
conf2.setBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, true);
conf2.setClass("fs.mocktrashfs.impl", MockTrashRootFS.class,
FileSystem.class);
ConfigUtil.addLink(conf2, "/mnt", URI.create("mocktrashfs://mnt/path"));
Path testPath = new Path(MockTrashRootFS.TRASH, "projs/proj");
FileSystem fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2);
Assert.assertEquals(MockTrashRootFS.TRASH, fsView2.getTrashRoot(testPath));
}
/**
* Test localized trash roots in getTrashRoots() for all users.
*/
@Test
public void testTrashRootsAllUsers() throws IOException {
Configuration conf2 = new Configuration(conf);
conf2.setBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, true);
FileSystem fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2);
// Case 1: verify correct trash roots from fsView and fsView2
int beforeTrashRootNum = fsView.getTrashRoots(true).size();
int beforeTrashRootNum2 = fsView2.getTrashRoots(true).size();
Assert.assertEquals(beforeTrashRootNum, beforeTrashRootNum2);
fsView.mkdirs(new Path("/data/" + TRASH_PREFIX + "/user1"));
fsView.mkdirs(new Path("/data/" + TRASH_PREFIX + "/user2"));
fsView.mkdirs(new Path("/user/" + TRASH_PREFIX + "/user3"));
fsView.mkdirs(new Path("/user/" + TRASH_PREFIX + "/user4"));
fsView.mkdirs(new Path("/user2/" + TRASH_PREFIX + "/user5"));
int afterTrashRootsNum = fsView.getTrashRoots(true).size();
int afterTrashRootsNum2 = fsView2.getTrashRoots(true).size();
Assert.assertEquals(beforeTrashRootNum, afterTrashRootsNum);
Assert.assertEquals(beforeTrashRootNum2 + 5, afterTrashRootsNum2);
// Case 2: per-user mount point
fsTarget.mkdirs(new Path(targetTestRoot, "Users/userA/.Trash/userA"));
Configuration conf3 = new Configuration(conf2);
ConfigUtil.addLink(conf3, "/Users/userA",
new Path(targetTestRoot, "Users/userA").toUri());
FileSystem fsView3 = FileSystem.get(FsConstants.VIEWFS_URI, conf3);
int trashRootsNum3 = fsView3.getTrashRoots(true).size();
Assert.assertEquals(afterTrashRootsNum2 + 1, trashRootsNum3);
// Case 3: single /Users mount point for all users
fsTarget.mkdirs(new Path(targetTestRoot, "Users/.Trash/user1"));
fsTarget.mkdirs(new Path(targetTestRoot, "Users/.Trash/user2"));
Configuration conf4 = new Configuration(conf2);
ConfigUtil.addLink(conf4, "/Users",
new Path(targetTestRoot, "Users").toUri());
FileSystem fsView4 = FileSystem.get(FsConstants.VIEWFS_URI, conf4);
int trashRootsNum4 = fsView4.getTrashRoots(true).size();
Assert.assertEquals(afterTrashRootsNum2 + 2, trashRootsNum4);
}
/**
* Test localized trash roots in getTrashRoots() for current user.
*/
@Test
public void testTrashRootsCurrentUser() throws IOException {
String currentUser =
UserGroupInformation.getCurrentUser().getShortUserName();
Configuration conf2 = new Configuration(conf);
conf2.setBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, true);
FileSystem fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2);
int beforeTrashRootNum = fsView.getTrashRoots(false).size();
int beforeTrashRootNum2 = fsView2.getTrashRoots(false).size();
Assert.assertEquals(beforeTrashRootNum, beforeTrashRootNum2);
fsView.mkdirs(new Path("/data/" + TRASH_PREFIX + "/" + currentUser));
fsView.mkdirs(new Path("/data/" + TRASH_PREFIX + "/user2"));
fsView.mkdirs(new Path("/user/" + TRASH_PREFIX + "/" + currentUser));
fsView.mkdirs(new Path("/user/" + TRASH_PREFIX + "/user4"));
fsView.mkdirs(new Path("/user2/" + TRASH_PREFIX + "/user5"));
int afterTrashRootsNum = fsView.getTrashRoots(false).size();
int afterTrashRootsNum2 = fsView2.getTrashRoots(false).size();
Assert.assertEquals(beforeTrashRootNum, afterTrashRootsNum);
Assert.assertEquals(beforeTrashRootNum2 + 2, afterTrashRootsNum2);
}
@Test(expected = NotInMountpointException.class)
public void testViewFileSystemUtil() throws Exception {
Configuration newConf = new Configuration(conf);