From 7e521c5a49fbcf88285c102051ea2522edc847b9 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Wed, 2 Nov 2016 19:11:05 -0700 Subject: [PATCH] HDFS-11080. Update HttpFS to use ConfigRedactor. Contributed by Sean Mackrory. --- .../hadoop/fs/CommonConfigurationKeysPublic.java | 2 ++ .../src/main/resources/core-default.xml | 2 +- .../apache/hadoop/conf/TestConfigRedactor.java | 2 ++ .../java/org/apache/hadoop/lib/server/Server.java | 15 ++++++--------- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index b5b107c2b3..f23dd5137a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -770,7 +770,9 @@ public class CommonConfigurationKeysPublic { public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS = "hadoop.security.sensitive-config-keys"; public static final String HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS_DEFAULT = + "secret$" + "," + "password$" + "," + + "ssl.keystore.pass$" + "," + "fs.s3.*[Ss]ecret.?[Kk]ey" + "," + "fs.azure\\.account.key.*" + "," + "dfs.webhdfs.oauth2.[a-z]+.token" + "," + diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 1e15b8ef09..327acfa4ba 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -504,7 +504,7 @@ hadoop.security.sensitive-config-keys - password$,fs.s3.*[Ss]ecret.?[Kk]ey,fs.azure.account.key.*,dfs.webhdfs.oauth2.[a-z]+.token,hadoop.security.sensitive-config-keys + secret$,password$,ssl.keystore.pass$,fs.s3.*[Ss]ecret.?[Kk]ey,fs.azure.account.key.*,dfs.webhdfs.oauth2.[a-z]+.token,hadoop.security.sensitive-config-keys A comma-separated list of regular expressions to match against configuration keys that should be redacted where appropriate, for example, when logging modified properties during a reconfiguration, diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java index 81f8f71795..eedb9b2b7c 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigRedactor.java @@ -47,6 +47,7 @@ public void redact() throws Exception { "dfs.webhdfs.oauth2.refresh.token", "ssl.server.keystore.keypassword", "ssl.server.keystore.password", + "httpfs.ssl.keystore.pass", "hadoop.security.sensitive-config-keys" ); for (String key : sensitiveKeys) { @@ -60,6 +61,7 @@ public void redact() throws Exception { "fs.defaultFS", "dfs.replication", "ssl.server.keystore.location", + "httpfs.config.dir", "hadoop.security.credstore.java-keystore-provider.password-file" ); for (String key : normalKeys) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/Server.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/Server.java index 1a0f9ff44e..82be027689 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/Server.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/server/Server.java @@ -19,6 +19,7 @@ package org.apache.hadoop.lib.server; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.ConfigRedactor; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.lib.util.Check; import org.apache.hadoop.lib.util.ConfigurationUtils; @@ -482,15 +483,13 @@ protected void initConfig() throws ServerException { } ConfigurationUtils.injectDefaults(defaultConf, config); - + ConfigRedactor redactor = new ConfigRedactor(config); for (String name : System.getProperties().stringPropertyNames()) { String value = System.getProperty(name); if (name.startsWith(getPrefix() + ".")) { config.set(name, value); - if (name.endsWith(".password") || name.endsWith(".secret")) { - value = "*MASKED*"; - } - log.info("System property sets {}: {}", name, value); + String redacted = redactor.redact(name, value); + log.info("System property sets {}: {}", name, redacted); } } @@ -499,10 +498,8 @@ protected void initConfig() throws ServerException { for (Map.Entry entry : config) { String name = entry.getKey(); String value = config.get(entry.getKey()); - if (name.endsWith(".password") || name.endsWith(".secret")) { - value = "*MASKED*"; - } - log.debug(" {}: {}", entry.getKey(), value); + String redacted = redactor.redact(name, value); + log.debug(" {}: {}", entry.getKey(), redacted); } log.debug("------------------------------------------------------"); }