From 582648befaf9908159f937d2cc8f549583a3483e Mon Sep 17 00:00:00 2001 From: John Zhuge Date: Thu, 10 Aug 2017 16:28:22 -0700 Subject: [PATCH] HADOOP-14260. Configuration.dumpConfiguration should redact sensitive information. Contributed by John Zhuge. --- .../org/apache/hadoop/conf/Configuration.java | 15 +++--- .../apache/hadoop/conf/TestConfiguration.java | 48 +++++++++++++++++-- 2 files changed, 53 insertions(+), 10 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index 65e8569ae2..edaee687b2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -3146,7 +3146,8 @@ public static void dumpConfiguration(Configuration config, JsonGenerator dumpGenerator = dumpFactory.createGenerator(out); dumpGenerator.writeStartObject(); dumpGenerator.writeFieldName("property"); - appendJSONProperty(dumpGenerator, config, propertyName); + appendJSONProperty(dumpGenerator, config, propertyName, + new ConfigRedactor(config)); dumpGenerator.writeEndObject(); dumpGenerator.flush(); } @@ -3186,11 +3187,11 @@ public static void dumpConfiguration(Configuration config, dumpGenerator.writeFieldName("properties"); dumpGenerator.writeStartArray(); dumpGenerator.flush(); + ConfigRedactor redactor = new ConfigRedactor(config); synchronized (config) { for (Map.Entry item: config.getProps().entrySet()) { - appendJSONProperty(dumpGenerator, - config, - item.getKey().toString()); + appendJSONProperty(dumpGenerator, config, item.getKey().toString(), + redactor); } } dumpGenerator.writeEndArray(); @@ -3208,12 +3209,14 @@ public static void dumpConfiguration(Configuration config, * @throws IOException */ private static void appendJSONProperty(JsonGenerator jsonGen, - Configuration config, String name) throws IOException { + Configuration config, String name, ConfigRedactor redactor) + throws IOException { // skip writing if given property name is empty or null if(!Strings.isNullOrEmpty(name) && jsonGen != null) { jsonGen.writeStartObject(); jsonGen.writeStringField("key", name); - jsonGen.writeStringField("value", config.get(name)); + jsonGen.writeStringField("value", + redactor.redact(name, config.get(name))); jsonGen.writeBooleanField("isFinal", config.finalParameters.contains(name)); String[] resources = config.updatingResource.get(name); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java index 92d3290103..91f25fa1ca 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java @@ -49,6 +49,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration.IntegerRanges; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.net.NetUtils; @@ -82,6 +83,11 @@ public class TestConfiguration extends TestCase { /** Four apostrophes. */ public static final String ESCAPED = "''''"; + private static final String SENSITIVE_CONFIG_KEYS = + CommonConfigurationKeysPublic.HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS; + + private BufferedWriter out; + @Override protected void setUp() throws Exception { super.setUp(); @@ -90,6 +96,9 @@ protected void setUp() throws Exception { @Override protected void tearDown() throws Exception { + if(out != null) { + out.close(); + } super.tearDown(); new File(CONFIG).delete(); new File(CONFIG2).delete(); @@ -878,8 +887,6 @@ public void testRelativeIncludes() throws Exception { new File(new File(relConfig).getParent()).delete(); } - BufferedWriter out; - public void testIntegerRanges() { Configuration conf = new Configuration(); conf.set("first", "-100"); @@ -1787,8 +1794,41 @@ public void testDumpConfiguratioWithoutDefaults() throws IOException { assertEquals(fileResource.toString(),prop.getResource()); } } - - + + public void testDumpSensitiveProperty() throws IOException { + final String myPassword = "ThisIsMyPassword"; + Configuration testConf = new Configuration(false); + out = new BufferedWriter(new FileWriter(CONFIG)); + startConfig(); + appendProperty("test.password", myPassword); + endConfig(); + Path fileResource = new Path(CONFIG); + testConf.addResource(fileResource); + + try (StringWriter outWriter = new StringWriter()) { + testConf.set(SENSITIVE_CONFIG_KEYS, "password$"); + Configuration.dumpConfiguration(testConf, "test.password", outWriter); + assertFalse(outWriter.toString().contains(myPassword)); + } + } + + public void testDumpSensitiveConfiguration() throws IOException { + final String myPassword = "ThisIsMyPassword"; + Configuration testConf = new Configuration(false); + out = new BufferedWriter(new FileWriter(CONFIG)); + startConfig(); + appendProperty("test.password", myPassword); + endConfig(); + Path fileResource = new Path(CONFIG); + testConf.addResource(fileResource); + + try (StringWriter outWriter = new StringWriter()) { + testConf.set(SENSITIVE_CONFIG_KEYS, "password$"); + Configuration.dumpConfiguration(testConf, outWriter); + assertFalse(outWriter.toString().contains(myPassword)); + } + } + public void testGetValByRegex() { Configuration conf = new Configuration(); String key1 = "t.abc.key1";