big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-15394. Add all available fs.viewfs.overload.scheme.target.<scheme>.impl classes in core-default.xml bydefault. Contributed by Uma Maheswara Rao G.

Browse Source 
(cherry picked from commit 3ca15292c5)
This commit is contained in:
Uma Maheswara Rao G 2020-06-06 08:11:57 -07:00
parent 7b175739a9
commit f85ce2570e
3 changed files with 136 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
View File

@ -952,6 +952,116 @@
</description>
</property>
<property>
<name>fs.viewfs.overload.scheme.target.hdfs.impl</name>
<value>org.apache.hadoop.hdfs.DistributedFileSystem</value>
<description>The DistributedFileSystem for view file system overload scheme
when child file system and ViewFSOverloadScheme's schemes are hdfs.
</description>
</property>
<property>
<name>fs.viewfs.overload.scheme.target.s3a.impl</name>
<value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
<description>The S3AFileSystem for view file system overload scheme when
child file system and ViewFSOverloadScheme's schemes are s3a.</description>
</property>
<property>
<name>fs.viewfs.overload.scheme.target.o3fs.impl</name>
<value>org.apache.hadoop.fs.ozone.OzoneFileSystem</value>
<description>The OzoneFileSystem for view file system overload scheme when
child file system and ViewFSOverloadScheme's schemes are o3fs.</description>
</property>
<property>
<name>fs.viewfs.overload.scheme.target.ftp.impl</name>
<value>org.apache.hadoop.fs.ftp.FTPFileSystem</value>
<description>The FTPFileSystem for view file system overload scheme when
child file system and ViewFSOverloadScheme's schemes are ftp.
</description>
</property>
<property>
<name>fs.viewfs.overload.scheme.target.webhdfs.impl</name>
<value>org.apache.hadoop.hdfs.web.WebHdfsFileSystem</value>
<description>The WebHdfsFileSystem for view file system overload scheme when
child file system and ViewFSOverloadScheme's schemes are webhdfs.
</description>
</property>
<property>
<name>fs.viewfs.overload.scheme.target.swebhdfs.impl</name>
<value>org.apache.hadoop.hdfs.web.SWebHdfsFileSystem</value>
<description>The SWebHdfsFileSystem for view file system overload scheme when
child file system and ViewFSOverloadScheme's schemes are swebhdfs.
</description>
</property>
<property>
<name>fs.viewfs.overload.scheme.target.file.impl</name>
<value>org.apache.hadoop.fs.LocalFileSystem</value>
<description>The LocalFileSystem for view file system overload scheme when
child file system and ViewFSOverloadScheme's schemes are file.
</description>
</property>
<property>
<name>fs.viewfs.overload.scheme.target.abfs.impl</name>
<value>org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem</value>
<description>The AzureBlobFileSystem for view file system overload scheme
when child file system and ViewFSOverloadScheme's schemes are abfs.
</description>
</property>
<property>
<name>fs.viewfs.overload.scheme.target.abfss.impl</name>
<value>org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem</value>
<description>The SecureAzureBlobFileSystem for view file system overload
scheme when child file system and ViewFSOverloadScheme's schemes are abfss.
</description>
</property>
<property>
<name>fs.viewfs.overload.scheme.target.wasb.impl</name>
<value>org.apache.hadoop.fs.azure.NativeAzureFileSystem</value>
<description>The NativeAzureFileSystem for view file system overload scheme
when child file system and ViewFSOverloadScheme's schemes are wasb.
</description>
</property>
<property>
<name>fs.viewfs.overload.scheme.target.swift.impl</name>
<value>org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem</value>
<description>The SwiftNativeFileSystem for view file system overload scheme
when child file system and ViewFSOverloadScheme's schemes are swift.
</description>
</property>
<property>
<name>fs.viewfs.overload.scheme.target.oss.impl</name>
<value>org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem</value>
<description>The AliyunOSSFileSystem for view file system overload scheme
when child file system and ViewFSOverloadScheme's schemes are oss.
</description>
</property>
<property>
<name>fs.viewfs.overload.scheme.target.http.impl</name>
<value>org.apache.hadoop.fs.http.HttpFileSystem</value>
<description>The HttpFileSystem for view file system overload scheme
when child file system and ViewFSOverloadScheme's schemes are http.
</description>
</property>
<property>
<name>fs.viewfs.overload.scheme.target.https.impl</name>
<value>org.apache.hadoop.fs.http.HttpsFileSystem</value>
<description>The HttpsFileSystem for view file system overload scheme
when child file system and ViewFSOverloadScheme's schemes are https.
</description>
</property>
<property>
<name>fs.AbstractFileSystem.ftp.impl</name>
<value>org.apache.hadoop.fs.ftp.FtpFs</value>

18
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
View File

@ -124,6 +124,24 @@ public void initializeMemberVariables() {
xmlPrefixToSkipCompare.add("fs.adl.");
xmlPropsToSkipCompare.add("fs.AbstractFileSystem.adl.impl");
// ViewfsOverloadScheme target fs impl property keys are dynamically
// constructed and they are advanced props.
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.abfs.impl");
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.abfss.impl");
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.file.impl");
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.ftp.impl");
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.hdfs.impl");
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.http.impl");
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.https.impl");
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.o3fs.impl");
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.oss.impl");
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.s3a.impl");
xmlPropsToSkipCompare.
add("fs.viewfs.overload.scheme.target.swebhdfs.impl");
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.webhdfs.impl");
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.wasb.impl");
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.swift.impl");
// Azure properties are in a different class
// - org.apache.hadoop.fs.azure.AzureNativeFileSystemStore
// - org.apache.hadoop.fs.azure.SASKeyGeneratorImpl

9
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java
View File

@ -382,13 +382,17 @@ public void testCreateOnRootShouldFailEvenFallBackMountLinkConfigured()
* Note: Above links created because to make fs initialization success.
* Otherwise will not proceed if no mount links.
*
* Don't set fs.viewfs.overload.scheme.target.hdfs.impl property.
* Unset fs.viewfs.overload.scheme.target.hdfs.impl property.
* So, OverloadScheme target fs initialization will fail.
*/
@Test(expected = UnsupportedFileSystemException.class, timeout = 30000)
public void testInvalidOverloadSchemeTargetFS() throws Exception {
final Path hdfsTargetPath = new Path(defaultFSURI + HDFS_USER_FOLDER);
String mountTableIfSet = conf.get(Constants.CONFIG_VIEWFS_MOUNTTABLE_PATH);
conf = new Configuration();
if (mountTableIfSet != null) {
conf.set(Constants.CONFIG_VIEWFS_MOUNTTABLE_PATH, mountTableIfSet);
}
addMountLinks(defaultFSURI.getAuthority(),
new String[] {HDFS_USER_FOLDER, LOCAL_FOLDER,
Constants.CONFIG_VIEWFS_LINK_FALLBACK },
@ -400,6 +404,9 @@ public void testInvalidOverloadSchemeTargetFS() throws Exception {
defaultFSURI.toString());
conf.set(String.format(FS_IMPL_PATTERN_KEY, HDFS_SCHEME),
ViewFileSystemOverloadScheme.class.getName());
conf.unset(String.format(
FsConstants.FS_VIEWFS_OVERLOAD_SCHEME_TARGET_FS_IMPL_PATTERN,
HDFS_SCHEME));
try (FileSystem fs = FileSystem.get(conf)) {
fs.createNewFile(new Path("/onRootWhenFallBack"));