From 788e7473a404fa074b3af522416ee3d2fae865a0 Mon Sep 17 00:00:00 2001 From: Surendra Singh Lilhore Date: Mon, 17 Dec 2018 11:24:57 +0530 Subject: [PATCH] HDFS-14096. [SPS] : Add Support for Storage Policy Satisfier in ViewFs. Contributed by Ayush Saxena. --- .../apache/hadoop/fs/AbstractFileSystem.java | 10 ++++++++++ .../java/org/apache/hadoop/fs/FileContext.java | 18 ++++++++++++++++++ .../java/org/apache/hadoop/fs/FileSystem.java | 10 ++++++++++ .../org/apache/hadoop/fs/FilterFileSystem.java | 5 +++++ .../java/org/apache/hadoop/fs/FilterFs.java | 5 +++++ .../hadoop/fs/viewfs/ChRootedFileSystem.java | 5 +++++ .../apache/hadoop/fs/viewfs/ChRootedFs.java | 5 +++++ .../hadoop/fs/viewfs/ViewFileSystem.java | 13 +++++++++++++ .../org/apache/hadoop/fs/viewfs/ViewFs.java | 12 ++++++++++++ .../apache/hadoop/fs/TestHarFileSystem.java | 2 ++ .../fs/viewfs/ViewFileSystemBaseTest.java | 5 +++++ .../main/java/org/apache/hadoop/fs/Hdfs.java | 5 +++++ .../hadoop/hdfs/DistributedFileSystem.java | 6 +----- 13 files changed, 96 insertions(+), 5 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java index 721f009bfa..d1e50d03d1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java @@ -1254,6 +1254,16 @@ public void deleteSnapshot(final Path snapshotDir, final String snapshotName) + " doesn't support deleteSnapshot"); } + /** + * Set the source path to satisfy storage policy. + * @param path The source path referring to either a directory or a file. + * @throws IOException + */ + public void satisfyStoragePolicy(final Path path) throws IOException { + throw new UnsupportedOperationException( + getClass().getSimpleName() + " doesn't support satisfyStoragePolicy"); + } + /** * Set the storage policy for a given file or directory. * diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java index 084f8f1c7b..e5438f553a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java @@ -2780,6 +2780,24 @@ public Void next(final AbstractFileSystem fs, final Path p) }.resolve(this, absF); } + /** + * Set the source path to satisfy storage policy. + * @param path The source path referring to either a directory or a file. + * @throws IOException + */ + public void satisfyStoragePolicy(final Path path) + throws IOException { + final Path absF = fixRelativePart(path); + new FSLinkResolver() { + @Override + public Void next(final AbstractFileSystem fs, final Path p) + throws IOException { + fs.satisfyStoragePolicy(path); + return null; + } + }.resolve(this, absF); + } + /** * Set the storage policy for a given file or directory. * diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index fe4159b617..5454cd0270 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -3085,6 +3085,16 @@ public void removeXAttr(Path path, String name) throws IOException { + " doesn't support removeXAttr"); } + /** + * Set the source path to satisfy storage policy. + * @param path The source path referring to either a directory or a file. + * @throws IOException + */ + public void satisfyStoragePolicy(final Path path) throws IOException { + throw new UnsupportedOperationException( + getClass().getSimpleName() + " doesn't support setStoragePolicy"); + } + /** * Set the storage policy for a given file or directory. * diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java index f9bbfb164b..a3e10b4fed 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java @@ -645,6 +645,11 @@ public void removeXAttr(Path path, String name) throws IOException { fs.removeXAttr(path, name); } + @Override + public void satisfyStoragePolicy(Path src) throws IOException { + fs.satisfyStoragePolicy(src); + } + @Override public void setStoragePolicy(Path src, String policyName) throws IOException { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java index b2a9aab75d..d6dc59bad3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java @@ -405,6 +405,11 @@ public void deleteSnapshot(final Path path, final String snapshotName) myFs.deleteSnapshot(path, snapshotName); } + @Override + public void satisfyStoragePolicy(final Path path) throws IOException { + myFs.satisfyStoragePolicy(path); + } + @Override public void setStoragePolicy(Path path, String policyName) throws IOException { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java index 8b90f533f8..b195be3297 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java @@ -449,6 +449,11 @@ public BlockStoragePolicySpi getStoragePolicy(Path src) throws IOException { return super.getStoragePolicy(fullPath(src)); } + @Override + public void satisfyStoragePolicy(Path src) throws IOException { + super.satisfyStoragePolicy(fullPath(src)); + } + @Override public void setStoragePolicy(Path src, String policyName) throws IOException { super.setStoragePolicy(fullPath(src), policyName); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java index 6168df0d09..5e14be15b9 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java @@ -398,6 +398,11 @@ public void deleteSnapshot(Path snapshotDir, String snapshotName) myFs.deleteSnapshot(fullPath(snapshotDir), snapshotName); } + @Override + public void satisfyStoragePolicy(final Path path) throws IOException { + myFs.satisfyStoragePolicy(path); + } + @Override public void setStoragePolicy(Path path, String policyName) throws IOException { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index 9523070760..bc73f1b753 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -810,6 +810,13 @@ public void deleteSnapshot(Path path, String snapshotName) res.targetFileSystem.deleteSnapshot(res.remainingPath, snapshotName); } + @Override + public void satisfyStoragePolicy(Path src) throws IOException { + InodeTree.ResolveResult res = + fsState.resolve(getUriPath(src), true); + res.targetFileSystem.satisfyStoragePolicy(res.remainingPath); + } + @Override public void setStoragePolicy(Path src, String policyName) throws IOException { InodeTree.ResolveResult res = fsState.resolve(getUriPath(src), @@ -1245,6 +1252,12 @@ public QuotaUsage getQuotaUsage(Path f) throws IOException { throw new NotInMountpointException(f, "getQuotaUsage"); } + @Override + public void satisfyStoragePolicy(Path src) throws IOException { + checkPathIsSlash(src); + throw readOnlyMountTable("satisfyStoragePolicy", src); + } + @Override public void setStoragePolicy(Path src, String policyName) throws IOException { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java index 8366eac90a..177047601e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java @@ -751,6 +751,13 @@ public void deleteSnapshot(Path path, String snapshotName) throws IOException { res.targetFileSystem.deleteSnapshot(res.remainingPath, snapshotName); } + @Override + public void satisfyStoragePolicy(final Path path) throws IOException { + InodeTree.ResolveResult res = + fsState.resolve(getUriPath(path), true); + res.targetFileSystem.satisfyStoragePolicy(res.remainingPath); + } + @Override public void setStoragePolicy(final Path path, final String policyName) throws IOException { @@ -1154,6 +1161,11 @@ public void deleteSnapshot(Path path, String snapshotName) throw readOnlyMountTable("deleteSnapshot", path); } + @Override + public void satisfyStoragePolicy(final Path path) throws IOException { + throw readOnlyMountTable("satisfyStoragePolicy", path); + } + @Override public void setStoragePolicy(Path path, String policyName) throws IOException { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java index 025b8314fb..b3021a9be7 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java @@ -213,6 +213,8 @@ public Map getXAttrs(Path path, List names) public void access(Path path, FsAction mode) throws IOException; + void satisfyStoragePolicy(Path src) throws IOException; + public void setStoragePolicy(Path src, String policyName) throws IOException; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java index b6158dfc6d..b9453df569 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java @@ -968,6 +968,11 @@ public void testInternalUnsetStoragePolicy() throws IOException { fsView.unsetStoragePolicy(new Path("/internalDir")); } + @Test(expected = AccessControlException.class) + public void testInternalSatisfyStoragePolicy() throws IOException { + fsView.satisfyStoragePolicy(new Path("/internalDir")); + } + @Test(expected = NotInMountpointException.class) public void testInternalgetStoragePolicy() throws IOException { fsView.getStoragePolicy(new Path("/internalDir")); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java index d07d5a2644..290f2c0e67 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java @@ -489,6 +489,11 @@ public void access(Path path, final FsAction mode) throws IOException { dfs.checkAccess(getUriPath(path), mode); } + @Override + public void satisfyStoragePolicy(Path path) throws IOException { + dfs.satisfyStoragePolicy(getUriPath(path)); + } + @Override public void setStoragePolicy(Path path, String policyName) throws IOException { dfs.setStoragePolicy(getUriPath(path), policyName); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 7dd02bde66..2b0b2c9f67 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -2870,11 +2870,7 @@ public Void next(final FileSystem fs, final Path p) throws IOException { } /** - * Set the source path to satisfy storage policy. This API is non-recursive - * in nature, i.e., if the source path is a directory then all the files - * immediately under the directory would be considered for satisfying the - * policy and the sub-directories if any under this path will be skipped. - * + * Set the source path to satisfy storage policy. * @param path The source path referring to either a directory or a file. * @throws IOException */