big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-17402. Add GCS config to the core-site (#2638)

Browse Source 
Contributed by Rafal Wojdyla
This commit is contained in:
Rafal Wojdyla 2021-07-05 16:07:12 -04:00 committed by GitHub
parent 93ad7c32f4
commit f639fbc29f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 21 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
View File

@ -685,6 +685,9 @@
fs.azure.account.key.* fs.azure.account.key.*
fs.azure.oauth2.* fs.azure.oauth2.*
fs.adl.oauth2.* fs.adl.oauth2.*
fs.gs.encryption.*
fs.gs.proxy.*
fs.gs.auth.*
credential$ credential$
oauth.*secret oauth.*secret
oauth.*password oauth.*password
@ -1071,6 +1074,15 @@
</description> </description>
</property> </property>
<property>
<name>fs.viewfs.overload.scheme.target.gs.impl</name>
<value>com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS</value>
<description>The GoogleHadoopFS/Google Cloud Storage file system for view
file system overload scheme when child file system and ViewFSOverloadScheme's
schemes are gs.
</description>
</property>
<property> <property>
<name>fs.viewfs.overload.scheme.target.https.impl</name> <name>fs.viewfs.overload.scheme.target.https.impl</name>
<value>org.apache.hadoop.fs.http.HttpsFileSystem</value> <value>org.apache.hadoop.fs.http.HttpsFileSystem</value>
@ -2264,6 +2276,13 @@
This setting provides better performance compared to blob-specific saskeys. This setting provides better performance compared to blob-specific saskeys.
</description> </description>
</property> </property>
<property>
<name>fs.AbstractFileSystem.gs.impl</name>
<value>com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS</value>
<description>The AbstractFileSystem for gs: uris.</description>
</property>
<property> <property>
<name>io.seqfile.compress.blocksize</name> <name>io.seqfile.compress.blocksize</name>
<value>1000000</value> <value>1000000</value>

4
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
View File

@ -117,7 +117,6 @@ public void initializeMemberVariables() {
xmlPrefixToSkipCompare.add("fs.abfs.impl"); xmlPrefixToSkipCompare.add("fs.abfs.impl");
xmlPrefixToSkipCompare.add("fs.abfss.impl"); xmlPrefixToSkipCompare.add("fs.abfss.impl");
// ADL properties are in a different subtree // ADL properties are in a different subtree
// - org.apache.hadoop.hdfs.web.ADLConfKeys // - org.apache.hadoop.hdfs.web.ADLConfKeys
xmlPrefixToSkipCompare.add("adl."); xmlPrefixToSkipCompare.add("adl.");
@ -130,6 +129,7 @@ public void initializeMemberVariables() {
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.abfss.impl"); xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.abfss.impl");
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.file.impl"); xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.file.impl");
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.ftp.impl"); xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.ftp.impl");
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.gs.impl");
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.hdfs.impl"); xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.hdfs.impl");
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.http.impl"); xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.http.impl");
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.https.impl"); xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.https.impl");
@ -139,9 +139,9 @@ public void initializeMemberVariables() {
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.s3a.impl"); xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.s3a.impl");
xmlPropsToSkipCompare. xmlPropsToSkipCompare.
add("fs.viewfs.overload.scheme.target.swebhdfs.impl"); add("fs.viewfs.overload.scheme.target.swebhdfs.impl");
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.swift.impl");
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.webhdfs.impl"); xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.webhdfs.impl");
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.wasb.impl"); xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.wasb.impl");
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.swift.impl");
// Azure properties are in a different class // Azure properties are in a different class
// - org.apache.hadoop.fs.azure.AzureNativeFileSystemStore // - org.apache.hadoop.fs.azure.AzureNativeFileSystemStore