big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-14260. Configuration.dumpConfiguration should redact sensitive information. Contributed by John Zhuge.

Browse Source
This commit is contained in:
John Zhuge 2017-08-10 16:28:22 -07:00 committed by John Zhuge
parent 4222c97108
commit 582648befa
2 changed files with 53 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
View File

@ -3146,7 +3146,8 @@ public static void dumpConfiguration(Configuration config,
JsonGenerator dumpGenerator = dumpFactory.createGenerator(out);
dumpGenerator.writeStartObject();
dumpGenerator.writeFieldName("property");
appendJSONProperty(dumpGenerator, config, propertyName);
appendJSONProperty(dumpGenerator, config, propertyName,
new ConfigRedactor(config));
dumpGenerator.writeEndObject();
dumpGenerator.flush();
}
@ -3186,11 +3187,11 @@ public static void dumpConfiguration(Configuration config,
dumpGenerator.writeFieldName("properties");
dumpGenerator.writeStartArray();
dumpGenerator.flush();
ConfigRedactor redactor = new ConfigRedactor(config);
synchronized (config) {
for (Map.Entry<Object,Object> item: config.getProps().entrySet()) {
appendJSONProperty(dumpGenerator,
config,
item.getKey().toString());
appendJSONProperty(dumpGenerator, config, item.getKey().toString(),
redactor);
}
}
dumpGenerator.writeEndArray();
@ -3208,12 +3209,14 @@ public static void dumpConfiguration(Configuration config,
* @throws IOException
*/
private static void appendJSONProperty(JsonGenerator jsonGen,
Configuration config, String name) throws IOException {
Configuration config, String name, ConfigRedactor redactor)
throws IOException {
// skip writing if given property name is empty or null
if(!Strings.isNullOrEmpty(name) && jsonGen != null) {
jsonGen.writeStartObject();
jsonGen.writeStringField("key", name);
jsonGen.writeStringField("value", config.get(name));
jsonGen.writeStringField("value",
redactor.redact(name, config.get(name)));
jsonGen.writeBooleanField("isFinal",
config.finalParameters.contains(name));
String[] resources = config.updatingResource.get(name);

48
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
View File

@ -49,6 +49,7 @@
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration.IntegerRanges;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.NetUtils;
@ -82,6 +83,11 @@ public class TestConfiguration extends TestCase {
/** Four apostrophes. */
public static final String ESCAPED = "&apos;&#39;&#0039;&#x27;";
private static final String SENSITIVE_CONFIG_KEYS =
CommonConfigurationKeysPublic.HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS;
private BufferedWriter out;
@Override
protected void setUp() throws Exception {
super.setUp();
@ -90,6 +96,9 @@ protected void setUp() throws Exception {
@Override
protected void tearDown() throws Exception {
if(out != null) {
out.close();
}
super.tearDown();
new File(CONFIG).delete();
new File(CONFIG2).delete();
@ -878,8 +887,6 @@ public void testRelativeIncludes() throws Exception {
new File(new File(relConfig).getParent()).delete();
}
BufferedWriter out;
public void testIntegerRanges() {
Configuration conf = new Configuration();
conf.set("first", "-100");
@ -1787,8 +1794,41 @@ public void testDumpConfiguratioWithoutDefaults() throws IOException {
assertEquals(fileResource.toString(),prop.getResource());
}
}
public void testDumpSensitiveProperty() throws IOException {
final String myPassword = "ThisIsMyPassword";
Configuration testConf = new Configuration(false);
out = new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("test.password", myPassword);
endConfig();
Path fileResource = new Path(CONFIG);
testConf.addResource(fileResource);
try (StringWriter outWriter = new StringWriter()) {
testConf.set(SENSITIVE_CONFIG_KEYS, "password$");
Configuration.dumpConfiguration(testConf, "test.password", outWriter);
assertFalse(outWriter.toString().contains(myPassword));
}
}
public void testDumpSensitiveConfiguration() throws IOException {
final String myPassword = "ThisIsMyPassword";
Configuration testConf = new Configuration(false);
out = new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("test.password", myPassword);
endConfig();
Path fileResource = new Path(CONFIG);
testConf.addResource(fileResource);
try (StringWriter outWriter = new StringWriter()) {
testConf.set(SENSITIVE_CONFIG_KEYS, "password$");
Configuration.dumpConfiguration(testConf, outWriter);
assertFalse(outWriter.toString().contains(myPassword));
}
}
public void testGetValByRegex() {
Configuration conf = new Configuration();
String key1 = "t.abc.key1";