big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-11080. Update HttpFS to use ConfigRedactor. Contributed by Sean Mackrory.

Browse Source
This commit is contained in:
Andrew Wang 2016-11-02 19:11:05 -07:00
parent dcc07ad8c8
commit 7e521c5a49
4 changed files with 11 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
View File

@ -770,7 +770,9 @@ public class CommonConfigurationKeysPublic {
public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS = public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS =
"hadoop.security.sensitive-config-keys"; "hadoop.security.sensitive-config-keys";
public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT = public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT =
"secret$" + "," +
"password$" + "," + "password$" + "," +
"ssl.keystore.pass$" + "," +
"fs.s3.*[Ss]ecret.?[Kk]ey" + "," + "fs.s3.*[Ss]ecret.?[Kk]ey" + "," +
"fs.azure\\.account.key.*" + "," + "fs.azure\\.account.key.*" + "," +
"dfs.webhdfs.oauth2.[a-z]+.token" + "," + "dfs.webhdfs.oauth2.[a-z]+.token" + "," +

2
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
View File

@ -504,7 +504,7 @@
<property> <property>
<name>hadoop.security.sensitive-config-keys</name> <name>hadoop.security.sensitive-config-keys</name>
<value>password$,fs.s3.*[Ss]ecret.?[Kk]ey,fs.azure.account.key.*,dfs.webhdfs.oauth2.[a-z]+.token,hadoop.security.sensitive-config-keys</value> <value>secret$,password$,ssl.keystore.pass$,fs.s3.*[Ss]ecret.?[Kk]ey,fs.azure.account.key.*,dfs.webhdfs.oauth2.[a-z]+.token,hadoop.security.sensitive-config-keys</value>
<description>A comma-separated list of regular expressions to match against <description>A comma-separated list of regular expressions to match against
configuration keys that should be redacted where appropriate, for configuration keys that should be redacted where appropriate, for
example, when logging modified properties during a reconfiguration, example, when logging modified properties during a reconfiguration,

2
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java
View File

@ -47,6 +47,7 @@ public void redact() throws Exception {
"dfs.webhdfs.oauth2.refresh.token", "dfs.webhdfs.oauth2.refresh.token",
"ssl.server.keystore.keypassword", "ssl.server.keystore.keypassword",
"ssl.server.keystore.password", "ssl.server.keystore.password",
"httpfs.ssl.keystore.pass",
"hadoop.security.sensitive-config-keys" "hadoop.security.sensitive-config-keys"
); );
for (String key : sensitiveKeys) { for (String key : sensitiveKeys) {
@ -60,6 +61,7 @@ public void redact() throws Exception {
"fs.defaultFS", "fs.defaultFS",
"dfs.replication", "dfs.replication",
"ssl.server.keystore.location", "ssl.server.keystore.location",
"httpfs.config.dir",
"hadoop.security.credstore.java-keystore-provider.password-file" "hadoop.security.credstore.java-keystore-provider.password-file"
); );
for (String key : normalKeys) { for (String key : normalKeys) {

15
hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/Server.java
View File

@ -19,6 +19,7 @@
package org.apache.hadoop.lib.server; package org.apache.hadoop.lib.server;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.ConfigRedactor;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.lib.util.Check; import org.apache.hadoop.lib.util.Check;
import org.apache.hadoop.lib.util.ConfigurationUtils; import org.apache.hadoop.lib.util.ConfigurationUtils;
@ -482,15 +483,13 @@ protected void initConfig() throws ServerException {
} }
ConfigurationUtils.injectDefaults(defaultConf, config); ConfigurationUtils.injectDefaults(defaultConf, config);
ConfigRedactor redactor = new ConfigRedactor(config);
for (String name : System.getProperties().stringPropertyNames()) { for (String name : System.getProperties().stringPropertyNames()) {
String value = System.getProperty(name); String value = System.getProperty(name);
if (name.startsWith(getPrefix() + ".")) { if (name.startsWith(getPrefix() + ".")) {
config.set(name, value); config.set(name, value);
if (name.endsWith(".password") || name.endsWith(".secret")) { String redacted = redactor.redact(name, value);
value = "*MASKED*"; log.info("System property sets {}: {}", name, redacted);
}
log.info("System property sets {}: {}", name, value);
} }
} }
@ -499,10 +498,8 @@ protected void initConfig() throws ServerException {
for (Map.Entry<String, String> entry : config) { for (Map.Entry<String, String> entry : config) {
String name = entry.getKey(); String name = entry.getKey();
String value = config.get(entry.getKey()); String value = config.get(entry.getKey());
if (name.endsWith(".password") || name.endsWith(".secret")) { String redacted = redactor.redact(name, value);
value = "*MASKED*"; log.debug(" {}: {}", entry.getKey(), redacted);
}
log.debug(" {}: {}", entry.getKey(), value);
} }
log.debug("------------------------------------------------------"); log.debug("------------------------------------------------------");
} }