diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java index 8235e937bd..94b07973ac 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java @@ -132,4 +132,10 @@ public interface Constants { Class DEFAULT_MOUNT_TABLE_CONFIG_LOADER_IMPL = HCFSMountTableConfigLoader.class; + + /** + * Enable ViewFileSystem to return a trashRoot which is local to mount point. + */ + String CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH = "fs.viewfs.mount.point.local.trash"; + boolean CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH_DEFAULT = false; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index 538f03a130..5ff3c2b70a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -25,6 +25,8 @@ import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS; import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS_DEFAULT; import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555; +import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH; +import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH_DEFAULT; import java.util.function.Function; import java.io.FileNotFoundException; @@ -1130,16 +1132,49 @@ public Collection getAllStoragePolicies() * Get the trash root directory for current user when the path * specified is deleted. * + * If CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH is not set, return + * the default trash root from targetFS. + * + * When CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH is set to true, + * 1) If path p is in fallback FS or from the same mount point as the default + * trash root for targetFS, return the default trash root for targetFS. + * 2) else, return a trash root in the mounted targetFS + * (/mntpoint/.Trash/{user}) + * * @param path the trash root of the path to be determined. * @return the trash root path. */ @Override public Path getTrashRoot(Path path) { + boolean useMountPointLocalTrash = + config.getBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, + CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH_DEFAULT); + try { InodeTree.ResolveResult res = fsState.resolve(getUriPath(path), true); - return res.targetFileSystem.getTrashRoot(res.remainingPath); - } catch (Exception e) { + + Path trashRoot = res.targetFileSystem.getTrashRoot(res.remainingPath); + if (!useMountPointLocalTrash) { + return trashRoot; + } else { + // Path p is either in a mount point or in the fallback FS + + if (ROOT_PATH.equals(new Path(res.resolvedPath)) + || trashRoot.toUri().getPath().startsWith(res.resolvedPath)) { + // Path p is in the fallback FS or targetFileSystem.trashRoot is in + // the same mount point as Path p + return trashRoot; + } else { + // targetFileSystem.trashRoot is in a different mount point from + // Path p. Return the trash root for the mount point. + Path mountPointRoot = + res.targetFileSystem.getFileStatus(new Path("/")).getPath(); + return new Path(mountPointRoot, + TRASH_PREFIX + "/" + ugi.getShortUserName()); + } + } + } catch (IOException | IllegalArgumentException e) { throw new NotInMountpointException(path, "getTrashRoot"); } } @@ -1156,6 +1191,62 @@ public Collection getTrashRoots(boolean allUsers) { for (FileSystem fs : getChildFileSystems()) { trashRoots.addAll(fs.getTrashRoots(allUsers)); } + + // Add trash dirs for each mount point + boolean useMountPointLocalTrash = + config.getBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, + CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH_DEFAULT); + if (useMountPointLocalTrash) { + + Set currentTrashPaths = new HashSet<>(); + for (FileStatus file : trashRoots) { + currentTrashPaths.add(file.getPath()); + } + + MountPoint[] mountPoints = getMountPoints(); + try { + for (int i = 0; i < mountPoints.length; i++) { + Path trashRoot = makeQualified( + new Path(mountPoints[i].mountedOnPath + "/" + TRASH_PREFIX)); + + // Continue if trashRoot does not exist for this filesystem + if (!exists(trashRoot)) { + continue; + } + + InodeTree.ResolveResult res = + fsState.resolve(getUriPath(trashRoot), true); + + if (!allUsers) { + Path userTrash = + new Path("/" + TRASH_PREFIX + "/" + ugi.getShortUserName()); + try { + FileStatus file = res.targetFileSystem.getFileStatus(userTrash); + if (!currentTrashPaths.contains(file.getPath())) { + trashRoots.add(file); + currentTrashPaths.add(file.getPath()); + } + } catch (FileNotFoundException ignored) { + } + } else { + FileStatus[] targetFsTrashRoots = + res.targetFileSystem.listStatus(new Path("/" + TRASH_PREFIX)); + for (FileStatus file : targetFsTrashRoots) { + // skip if we already include it in currentTrashPaths + if (currentTrashPaths.contains(file.getPath())) { + continue; + } + + trashRoots.add(file); + currentTrashPaths.add(file.getPath()); + } + } + } + } catch (IOException e) { + LOG.warn("Exception in get all trash roots", e); + } + } + return trashRoots; } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java index 434eff0bef..91a90751e8 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java @@ -69,6 +69,8 @@ import static org.apache.hadoop.fs.FileSystemTestHelper.*; import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE; import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555; +import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH; +import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX; import org.junit.After; import org.junit.Assert; @@ -1102,6 +1104,160 @@ public void testTrashRoot() throws IOException { Assert.assertTrue("", fsView.getTrashRoots(true).size() > 0); } + /** + * Test the localized trash root for getTrashRoot. + */ + @Test + public void testTrashRootLocalizedTrash() throws IOException { + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + Configuration conf2 = new Configuration(conf); + conf2.setBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, true); + ConfigUtil.addLinkFallback(conf2, targetTestRoot.toUri()); + FileSystem fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2); + + // Case 1: path p not in the default FS. + // Return a trash root within the mount point. + Path dataTestPath = new Path("/data/dir/file"); + Path dataTrashRoot = new Path(targetTestRoot, + "data/" + TRASH_PREFIX + "/" + ugi.getShortUserName()); + Assert.assertEquals(dataTrashRoot, fsView2.getTrashRoot(dataTestPath)); + + // Case 2: path p not found in mount table, fall back to the default FS + // Return a trash root in user home dir + Path nonExistentPath = new Path("/nonExistentDir/nonExistentFile"); + Path userTrashRoot = new Path(fsTarget.getHomeDirectory(), TRASH_PREFIX); + Assert.assertEquals(userTrashRoot, fsView2.getTrashRoot(nonExistentPath)); + + // Case 3: turn off the CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH flag. + // Return a trash root in user home dir. + conf2.setBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, false); + fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2); + Assert.assertEquals(userTrashRoot, fsView2.getTrashRoot(dataTestPath)); + + // Case 4: viewFS without fallback. Expect exception for a nonExistent path + conf2 = new Configuration(conf); + fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2); + try { + fsView2.getTrashRoot(nonExistentPath); + } catch (NotInMountpointException ignored) { + } + + // Case 5: path p is in the same mount point as targetFS.getTrashRoot(). + // Return targetFS.getTrashRoot() + // Use a new Configuration object, so that we can start with an empty + // mount table. This would avoid a conflict between the /user link in + // setupMountPoints() and homeDir we will need to setup for this test. + // default homeDir for hdfs is /user/. + Configuration conf3 = ViewFileSystemTestSetup.createConfig(); + Path homeDir = fsTarget.getHomeDirectory(); + String homeParentDir = homeDir.getParent().toUri().getPath(); + conf3.setBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, true); + ConfigUtil.addLink(conf3, homeParentDir, + new Path(targetTestRoot, homeParentDir).toUri()); + Path homeTestPath = new Path(homeDir.toUri().getPath(), "testuser/file"); + FileSystem fsView3 = FileSystem.get(FsConstants.VIEWFS_URI, conf3); + Assert.assertEquals(userTrashRoot, fsView3.getTrashRoot(homeTestPath)); + } + + /** + * A mocked FileSystem which returns a deep trash dir. + */ + static class MockTrashRootFS extends MockFileSystem { + public static final Path TRASH = + new Path("/mnt/very/deep/deep/trash/dir/.Trash"); + + @Override + public Path getTrashRoot(Path path) { + return TRASH; + } + } + + /** + * Test a trash root that is inside a mount point for getTrashRoot + */ + @Test + public void testTrashRootDeepTrashDir() throws IOException { + + Configuration conf2 = ViewFileSystemTestSetup.createConfig(); + conf2.setBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, true); + conf2.setClass("fs.mocktrashfs.impl", MockTrashRootFS.class, + FileSystem.class); + ConfigUtil.addLink(conf2, "/mnt", URI.create("mocktrashfs://mnt/path")); + Path testPath = new Path(MockTrashRootFS.TRASH, "projs/proj"); + FileSystem fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2); + Assert.assertEquals(MockTrashRootFS.TRASH, fsView2.getTrashRoot(testPath)); + } + + /** + * Test localized trash roots in getTrashRoots() for all users. + */ + @Test + public void testTrashRootsAllUsers() throws IOException { + Configuration conf2 = new Configuration(conf); + conf2.setBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, true); + FileSystem fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2); + + // Case 1: verify correct trash roots from fsView and fsView2 + int beforeTrashRootNum = fsView.getTrashRoots(true).size(); + int beforeTrashRootNum2 = fsView2.getTrashRoots(true).size(); + Assert.assertEquals(beforeTrashRootNum, beforeTrashRootNum2); + + fsView.mkdirs(new Path("/data/" + TRASH_PREFIX + "/user1")); + fsView.mkdirs(new Path("/data/" + TRASH_PREFIX + "/user2")); + fsView.mkdirs(new Path("/user/" + TRASH_PREFIX + "/user3")); + fsView.mkdirs(new Path("/user/" + TRASH_PREFIX + "/user4")); + fsView.mkdirs(new Path("/user2/" + TRASH_PREFIX + "/user5")); + int afterTrashRootsNum = fsView.getTrashRoots(true).size(); + int afterTrashRootsNum2 = fsView2.getTrashRoots(true).size(); + Assert.assertEquals(beforeTrashRootNum, afterTrashRootsNum); + Assert.assertEquals(beforeTrashRootNum2 + 5, afterTrashRootsNum2); + + // Case 2: per-user mount point + fsTarget.mkdirs(new Path(targetTestRoot, "Users/userA/.Trash/userA")); + Configuration conf3 = new Configuration(conf2); + ConfigUtil.addLink(conf3, "/Users/userA", + new Path(targetTestRoot, "Users/userA").toUri()); + FileSystem fsView3 = FileSystem.get(FsConstants.VIEWFS_URI, conf3); + int trashRootsNum3 = fsView3.getTrashRoots(true).size(); + Assert.assertEquals(afterTrashRootsNum2 + 1, trashRootsNum3); + + // Case 3: single /Users mount point for all users + fsTarget.mkdirs(new Path(targetTestRoot, "Users/.Trash/user1")); + fsTarget.mkdirs(new Path(targetTestRoot, "Users/.Trash/user2")); + Configuration conf4 = new Configuration(conf2); + ConfigUtil.addLink(conf4, "/Users", + new Path(targetTestRoot, "Users").toUri()); + FileSystem fsView4 = FileSystem.get(FsConstants.VIEWFS_URI, conf4); + int trashRootsNum4 = fsView4.getTrashRoots(true).size(); + Assert.assertEquals(afterTrashRootsNum2 + 2, trashRootsNum4); + } + + /** + * Test localized trash roots in getTrashRoots() for current user. + */ + @Test + public void testTrashRootsCurrentUser() throws IOException { + String currentUser = + UserGroupInformation.getCurrentUser().getShortUserName(); + Configuration conf2 = new Configuration(conf); + conf2.setBoolean(CONFIG_VIEWFS_MOUNT_POINT_LOCAL_TRASH, true); + FileSystem fsView2 = FileSystem.get(FsConstants.VIEWFS_URI, conf2); + + int beforeTrashRootNum = fsView.getTrashRoots(false).size(); + int beforeTrashRootNum2 = fsView2.getTrashRoots(false).size(); + Assert.assertEquals(beforeTrashRootNum, beforeTrashRootNum2); + + fsView.mkdirs(new Path("/data/" + TRASH_PREFIX + "/" + currentUser)); + fsView.mkdirs(new Path("/data/" + TRASH_PREFIX + "/user2")); + fsView.mkdirs(new Path("/user/" + TRASH_PREFIX + "/" + currentUser)); + fsView.mkdirs(new Path("/user/" + TRASH_PREFIX + "/user4")); + fsView.mkdirs(new Path("/user2/" + TRASH_PREFIX + "/user5")); + int afterTrashRootsNum = fsView.getTrashRoots(false).size(); + int afterTrashRootsNum2 = fsView2.getTrashRoots(false).size(); + Assert.assertEquals(beforeTrashRootNum, afterTrashRootsNum); + Assert.assertEquals(beforeTrashRootNum2 + 2, afterTrashRootsNum2); + } + @Test(expected = NotInMountpointException.class) public void testViewFileSystemUtil() throws Exception { Configuration newConf = new Configuration(conf);