big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-14096. [SPS] : Add Support for Storage Policy Satisfier in ViewFs. Contributed by Ayush Saxena.

Browse Source
This commit is contained in:
Surendra Singh Lilhore 2018-12-17 11:24:57 +05:30
parent 04c03476c9
commit 788e7473a4
13 changed files with 96 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
View File

@ -1254,6 +1254,16 @@ public void deleteSnapshot(final Path snapshotDir, final String snapshotName)
+ " doesn't support deleteSnapshot");
}
/**
* Set the source path to satisfy storage policy.
* @param path The source path referring to either a directory or a file.
* @throws IOException
*/
public void satisfyStoragePolicy(final Path path) throws IOException {
throw new UnsupportedOperationException(
getClass().getSimpleName() + " doesn't support satisfyStoragePolicy");
}
/**
* Set the storage policy for a given file or directory.
*

18
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
View File

@ -2780,6 +2780,24 @@ public Void next(final AbstractFileSystem fs, final Path p)
}.resolve(this, absF);
}
/**
* Set the source path to satisfy storage policy.
* @param path The source path referring to either a directory or a file.
* @throws IOException
*/
public void satisfyStoragePolicy(final Path path)
throws IOException {
final Path absF = fixRelativePart(path);
new FSLinkResolver<Void>() {
@Override
public Void next(final AbstractFileSystem fs, final Path p)
throws IOException {
fs.satisfyStoragePolicy(path);
return null;
}
}.resolve(this, absF);
}
/**
* Set the storage policy for a given file or directory.
*

10
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
View File

@ -3085,6 +3085,16 @@ public void removeXAttr(Path path, String name) throws IOException {
+ " doesn't support removeXAttr");
}
/**
* Set the source path to satisfy storage policy.
* @param path The source path referring to either a directory or a file.
* @throws IOException
*/
public void satisfyStoragePolicy(final Path path) throws IOException {
throw new UnsupportedOperationException(
getClass().getSimpleName() + " doesn't support setStoragePolicy");
}
/**
* Set the storage policy for a given file or directory.
*

5
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
View File

@ -645,6 +645,11 @@ public void removeXAttr(Path path, String name) throws IOException {
fs.removeXAttr(path, name);
}
@Override
public void satisfyStoragePolicy(Path src) throws IOException {
fs.satisfyStoragePolicy(src);
}
@Override
public void setStoragePolicy(Path src, String policyName)
throws IOException {

5
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
View File

@ -405,6 +405,11 @@ public void deleteSnapshot(final Path path, final String snapshotName)
myFs.deleteSnapshot(path, snapshotName);
}
@Override
public void satisfyStoragePolicy(final Path path) throws IOException {
myFs.satisfyStoragePolicy(path);
}
@Override
public void setStoragePolicy(Path path, String policyName)
throws IOException {

5
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
View File

@ -449,6 +449,11 @@ public BlockStoragePolicySpi getStoragePolicy(Path src) throws IOException {
return super.getStoragePolicy(fullPath(src));
}
@Override
public void satisfyStoragePolicy(Path src) throws IOException {
super.satisfyStoragePolicy(fullPath(src));
}
@Override
public void setStoragePolicy(Path src, String policyName) throws IOException {
super.setStoragePolicy(fullPath(src), policyName);

5
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java
View File

@ -398,6 +398,11 @@ public void deleteSnapshot(Path snapshotDir, String snapshotName)
myFs.deleteSnapshot(fullPath(snapshotDir), snapshotName);
}
@Override
public void satisfyStoragePolicy(final Path path) throws IOException {
myFs.satisfyStoragePolicy(path);
}
@Override
public void setStoragePolicy(Path path, String policyName)
throws IOException {

13
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
View File

@ -810,6 +810,13 @@ public void deleteSnapshot(Path path, String snapshotName)
res.targetFileSystem.deleteSnapshot(res.remainingPath, snapshotName);
}
@Override
public void satisfyStoragePolicy(Path src) throws IOException {
InodeTree.ResolveResult<FileSystem> res =
fsState.resolve(getUriPath(src), true);
res.targetFileSystem.satisfyStoragePolicy(res.remainingPath);
}
@Override
public void setStoragePolicy(Path src, String policyName) throws IOException {
InodeTree.ResolveResult<FileSystem> res = fsState.resolve(getUriPath(src),
@ -1245,6 +1252,12 @@ public QuotaUsage getQuotaUsage(Path f) throws IOException {
throw new NotInMountpointException(f, "getQuotaUsage");
}
@Override
public void satisfyStoragePolicy(Path src) throws IOException {
checkPathIsSlash(src);
throw readOnlyMountTable("satisfyStoragePolicy", src);
}
@Override
public void setStoragePolicy(Path src, String policyName)
throws IOException {

12
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
View File

@ -751,6 +751,13 @@ public void deleteSnapshot(Path path, String snapshotName) throws IOException {
res.targetFileSystem.deleteSnapshot(res.remainingPath, snapshotName);
}
@Override
public void satisfyStoragePolicy(final Path path) throws IOException {
InodeTree.ResolveResult<AbstractFileSystem> res =
fsState.resolve(getUriPath(path), true);
res.targetFileSystem.satisfyStoragePolicy(res.remainingPath);
}
@Override
public void setStoragePolicy(final Path path, final String policyName)
throws IOException {
@ -1154,6 +1161,11 @@ public void deleteSnapshot(Path path, String snapshotName)
throw readOnlyMountTable("deleteSnapshot", path);
}
@Override
public void satisfyStoragePolicy(final Path path) throws IOException {
throw readOnlyMountTable("satisfyStoragePolicy", path);
}
@Override
public void setStoragePolicy(Path path, String policyName)
throws IOException {

2
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
View File

@ -213,6 +213,8 @@ public Map<String, byte[]> getXAttrs(Path path, List<String> names)
public void access(Path path, FsAction mode) throws IOException;
void satisfyStoragePolicy(Path src) throws IOException;
public void setStoragePolicy(Path src, String policyName)
throws IOException;

5
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
View File

@ -968,6 +968,11 @@ public void testInternalUnsetStoragePolicy() throws IOException {
fsView.unsetStoragePolicy(new Path("/internalDir"));
}
@Test(expected = AccessControlException.class)
public void testInternalSatisfyStoragePolicy() throws IOException {
fsView.satisfyStoragePolicy(new Path("/internalDir"));
}
@Test(expected = NotInMountpointException.class)
public void testInternalgetStoragePolicy() throws IOException {
fsView.getStoragePolicy(new Path("/internalDir"));

5
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
View File

@ -489,6 +489,11 @@ public void access(Path path, final FsAction mode) throws IOException {
dfs.checkAccess(getUriPath(path), mode);
}
@Override
public void satisfyStoragePolicy(Path path) throws IOException {
dfs.satisfyStoragePolicy(getUriPath(path));
}
@Override
public void setStoragePolicy(Path path, String policyName) throws IOException {
dfs.setStoragePolicy(getUriPath(path), policyName);

6
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
View File

@ -2870,11 +2870,7 @@ public Void next(final FileSystem fs, final Path p) throws IOException {
}
/**
* Set the source path to satisfy storage policy. This API is non-recursive
* in nature, i.e., if the source path is a directory then all the files
* immediately under the directory would be considered for satisfying the
* policy and the sub-directories if any under this path will be skipped.
*
* Set the source path to satisfy storage policy.
* @param path The source path referring to either a directory or a file.
* @throws IOException
*/