big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-13272. ViewFileSystem should support storage policy related API. Contributed by Peter Shi

Browse Source
This commit is contained in:
Chris Douglas 2016-07-22 18:07:07 -07:00
parent 5fda66e4a3
commit 22a9a6b0d4
4 changed files with 201 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
View File

@ -27,6 +27,7 @@
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.BlockStoragePolicySpi;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataInputStream;
@ -39,12 +40,12 @@
import org.apache.hadoop.fs.FsStatus;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.util.Progressable;
@ -425,5 +426,21 @@ public FsServerDefaults getServerDefaults() throws IOException {
@Override
public FsServerDefaults getServerDefaults(Path f) throws IOException {
return super.getServerDefaults(fullPath(f));
}
}
@Override
public BlockStoragePolicySpi getStoragePolicy(Path src) throws IOException {
return super.getStoragePolicy(fullPath(src));
}
@Override
public void setStoragePolicy(Path src, String policyName) throws IOException {
super.setStoragePolicy(fullPath(src), policyName);
}
@Override
public void unsetStoragePolicy(Path src) throws IOException {
super.unsetStoragePolicy(fullPath(src));
}
}

75
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
View File

@ -24,6 +24,7 @@
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Arrays;
import java.util.Collection;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
@ -35,6 +36,7 @@
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.BlockStoragePolicySpi;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataInputStream;
@ -48,6 +50,7 @@
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.fs.XAttrSetFlag;
@ -56,7 +59,6 @@
import org.apache.hadoop.fs.permission.AclUtil;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.fs.viewfs.InodeTree.INode;
import org.apache.hadoop.fs.viewfs.InodeTree.INodeLink;
import org.apache.hadoop.security.AccessControlException;
@ -760,6 +762,43 @@ public void deleteSnapshot(Path path, String snapshotName)
res.targetFileSystem.deleteSnapshot(res.remainingPath, snapshotName);
}
@Override
public void setStoragePolicy(Path src, String policyName) throws IOException {
InodeTree.ResolveResult<FileSystem> res = fsState.resolve(getUriPath(src),
true);
res.targetFileSystem.setStoragePolicy(res.remainingPath, policyName);
}
@Override
public void unsetStoragePolicy(Path src) throws IOException {
InodeTree.ResolveResult<FileSystem> res = fsState.resolve(getUriPath(src),
true);
res.targetFileSystem.unsetStoragePolicy(res.remainingPath);
}
@Override
public BlockStoragePolicySpi getStoragePolicy(Path src) throws IOException {
InodeTree.ResolveResult<FileSystem> res = fsState.resolve(getUriPath(src),
true);
return res.targetFileSystem.getStoragePolicy(res.remainingPath);
}
@Override
public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
throws IOException {
Collection<BlockStoragePolicySpi> allPolicies = new HashSet<>();
for (FileSystem fs : getChildFileSystems()) {
try {
Collection<? extends BlockStoragePolicySpi> policies =
fs.getAllStoragePolicies();
allPolicies.addAll(policies);
} catch (UnsupportedOperationException e) {
// ignored
}
}
return allPolicies;
}
/*
* An instance of this class represents an internal dir of the viewFs
* that is internal dir of the mount table.
@ -1079,5 +1118,39 @@ public void deleteSnapshot(Path path, String snapshotName)
public QuotaUsage getQuotaUsage(Path f) throws IOException {
throw new NotInMountpointException(f, "getQuotaUsage");
}
@Override
public void setStoragePolicy(Path src, String policyName)
throws IOException {
checkPathIsSlash(src);
throw readOnlyMountTable("setStoragePolicy", src);
}
@Override
public void unsetStoragePolicy(Path src) throws IOException {
checkPathIsSlash(src);
throw readOnlyMountTable("unsetStoragePolicy", src);
}
@Override
public BlockStoragePolicySpi getStoragePolicy(Path src) throws IOException {
throw new NotInMountpointException(src, "getStoragePolicy");
}
@Override
public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
throws IOException {
Collection<BlockStoragePolicySpi> allPolicies = new HashSet<>();
for (FileSystem fs : getChildFileSystems()) {
try {
Collection<? extends BlockStoragePolicySpi> policies =
fs.getAllStoragePolicies();
allPolicies.addAll(policies);
} catch (UnsupportedOperationException e) {
// ignored
}
}
return allPolicies;
}
}
}

65
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestChRootedFileSystem.java
View File

@ -468,4 +468,69 @@ public void testRenameSnapshot() throws Exception {
verify(mockFs).renameSnapshot(chRootedSnapRootPath, "snapOldName",
"snapNewName");
}
@Test(timeout = 30000)
public void testSetStoragePolicy() throws Exception {
Path storagePolicyPath = new Path("/storagePolicy");
Path chRootedStoragePolicyPath = new Path("/a/b/storagePolicy");
Configuration conf = new Configuration();
conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
URI chrootUri = URI.create("mockfs://foo/a/b");
ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf);
FileSystem mockFs = ((FilterFileSystem) chrootFs.getRawFileSystem())
.getRawFileSystem();
chrootFs.setStoragePolicy(storagePolicyPath, "HOT");
verify(mockFs).setStoragePolicy(chRootedStoragePolicyPath, "HOT");
}
@Test(timeout = 30000)
public void testUnsetStoragePolicy() throws Exception {
Path storagePolicyPath = new Path("/storagePolicy");
Path chRootedStoragePolicyPath = new Path("/a/b/storagePolicy");
Configuration conf = new Configuration();
conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
URI chrootUri = URI.create("mockfs://foo/a/b");
ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf);
FileSystem mockFs = ((FilterFileSystem) chrootFs.getRawFileSystem())
.getRawFileSystem();
chrootFs.unsetStoragePolicy(storagePolicyPath);
verify(mockFs).unsetStoragePolicy(chRootedStoragePolicyPath);
}
@Test(timeout = 30000)
public void testGetStoragePolicy() throws Exception {
Path storagePolicyPath = new Path("/storagePolicy");
Path chRootedStoragePolicyPath = new Path("/a/b/storagePolicy");
Configuration conf = new Configuration();
conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
URI chrootUri = URI.create("mockfs://foo/a/b");
ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf);
FileSystem mockFs = ((FilterFileSystem) chrootFs.getRawFileSystem())
.getRawFileSystem();
chrootFs.getStoragePolicy(storagePolicyPath);
verify(mockFs).getStoragePolicy(chRootedStoragePolicyPath);
}
@Test(timeout = 30000)
public void testGetAllStoragePolicy() throws Exception {
Configuration conf = new Configuration();
conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
URI chrootUri = URI.create("mockfs://foo/a/b");
ChRootedFileSystem chrootFs = new ChRootedFileSystem(chrootUri, conf);
FileSystem mockFs = ((FilterFileSystem) chrootFs.getRawFileSystem())
.getRawFileSystem();
chrootFs.getAllStoragePolicies();
verify(mockFs).getAllStoragePolicies();
}
}

54
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
View File

@ -19,42 +19,41 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.ArrayList;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.BlockStoragePolicySpi;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileSystemTestHelper;
import static org.apache.hadoop.fs.FileSystemTestHelper.*;
import org.apache.hadoop.fs.permission.AclEntry;
import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FsConstants;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.AclUtil;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.viewfs.ConfigUtil;
import org.apache.hadoop.fs.viewfs.ViewFileSystem;
import org.apache.hadoop.fs.viewfs.ViewFileSystem.MountPoint;
import org.apache.hadoop.fs.viewfs.ViewFileSystem;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import static org.apache.hadoop.fs.FileSystemTestHelper.*;
import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.*;
/**
@ -885,4 +884,35 @@ public void testInternalRenameSnapshot() throws IOException {
public void testInternalDeleteSnapshot() throws IOException {
fsView.deleteSnapshot(new Path("/internalDir"), "snap1");
}
@Test(expected = AccessControlException.class)
public void testInternalSetStoragePolicy() throws IOException {
fsView.setStoragePolicy(new Path("/internalDir"), "HOT");
}
@Test(expected = AccessControlException.class)
public void testInternalUnsetStoragePolicy() throws IOException {
fsView.unsetStoragePolicy(new Path("/internalDir"));
}
@Test(expected = NotInMountpointException.class)
public void testInternalgetStoragePolicy() throws IOException {
fsView.getStoragePolicy(new Path("/internalDir"));
}
@Test
public void testInternalGetAllStoragePolicies() throws IOException {
Collection<? extends BlockStoragePolicySpi> policies =
fsView.getAllStoragePolicies();
for (FileSystem fs : fsView.getChildFileSystems()) {
try {
for (BlockStoragePolicySpi s : fs.getAllStoragePolicies()) {
assertTrue("Missing policy: " + s, policies.contains(s));
}
} catch (UnsupportedOperationException e) {
// ignore
}
}
}
}