From 280ab0cf7d71882cd3af8cf4e77a1af96844e0c6 Mon Sep 17 00:00:00 2001 From: Hemanth Yamijala Date: Mon, 24 Aug 2009 11:21:18 +0000 Subject: [PATCH] HADOOP-6184. Provide an API to dump Configuration in a JSON format. Contributed by V.V.Chaitanya Krishna. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@807149 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 3 + .../org/apache/hadoop/conf/Configuration.java | 82 +++++++- .../apache/hadoop/conf/TestConfiguration.java | 179 ++++++++++++++++++ 3 files changed, 262 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 06653dab8d..73e5040530 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -504,6 +504,9 @@ Trunk (unreleased changes) HADOOP-6173. Change src/native/packageNativeHadoop.sh to package all native library files. (Hong Tang via szetszwo) + + HADOOP-6184. Provide an API to dump Configuration in a JSON format. + (V.V.Chaitanya Krishna via yhemanth) OPTIMIZATIONS diff --git a/src/java/org/apache/hadoop/conf/Configuration.java b/src/java/org/apache/hadoop/conf/Configuration.java index 3873e66aa6..c53d33a299 100644 --- a/src/java/org/apache/hadoop/conf/Configuration.java +++ b/src/java/org/apache/hadoop/conf/Configuration.java @@ -28,6 +28,7 @@ import java.io.InputStreamReader; import java.io.OutputStream; import java.io.Reader; +import java.io.Writer; import java.net.URL; import java.util.ArrayList; import java.util.Collection; @@ -62,6 +63,8 @@ import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.StringUtils; +import org.codehaus.jackson.JsonFactory; +import org.codehaus.jackson.JsonGenerator; import org.w3c.dom.DOMException; import org.w3c.dom.Document; import org.w3c.dom.Element; @@ -155,7 +158,7 @@ public class Configuration implements Iterable>, private boolean loadDefaults = true; /** - * Configurtion objects + * Configuration objects */ private static final WeakHashMap REGISTRY = new WeakHashMap(); @@ -167,6 +170,18 @@ public class Configuration implements Iterable>, private static final ArrayList defaultResources = new ArrayList(); + /** + * Flag to indicate if the storage of resource which updates a key needs + * to be stored for each key + */ + private boolean storeResource; + + /** + * Stores the mapping of key to the resource which modifies or loads + * the key most recently + */ + private HashMap updatingResource; + static{ //print deprecation warning if hadoop-site.xml is found in classpath ClassLoader cL = Thread.currentThread().getContextClassLoader(); @@ -214,6 +229,23 @@ public Configuration(boolean loadDefaults) { synchronized(Configuration.class) { REGISTRY.put(this, null); } + this.storeResource = false; + } + + /** + * A new configuration with the same settings and additional facility for + * storage of resource to each key which loads or updates + * the key most recently + * @param other the configuration from which to clone settings + * @param storeResource flag to indicate if the storage of resource to + * each key is to be stored + */ + private Configuration(Configuration other, boolean storeResource) { + this(other); + this.storeResource = storeResource; + if (storeResource) { + updatingResource = new HashMap(); + } } /** @@ -1081,8 +1113,14 @@ protected synchronized Properties getProps() { if (properties == null) { properties = new Properties(); loadResources(properties, resources, quietmode); - if (overlay!= null) + if (overlay!= null) { properties.putAll(overlay); + if (storeResource) { + for (Map.Entry item: overlay.entrySet()) { + updatingResource.put((String) item.getKey(), "Unknown"); + } + } + } } return properties; } @@ -1251,6 +1289,9 @@ private void loadResource(Properties properties, Object name, boolean quiet) { if (attr != null && value != null) { if (!finalParameters.contains(attr)) { properties.setProperty(attr, value); + if (storeResource) { + updatingResource.put(attr, name.toString()); + } if (finalParameter) finalParameters.add(attr); } else { @@ -1322,6 +1363,43 @@ public void writeXml(OutputStream out) throws IOException { } } + /** + * Writes out all the parameters and their properties (final and resource) to + * the given {@link Writer} + * The format of the output would be + * { "properties" : [ {key1,value1,key1.isFinal,key1.resource}, {key2,value2, + * key2.isFinal,key2.resource}... ] } + * It does not output the parameters of the configuration object which is + * loaded from an input stream. + * @param out the Writer to write to + * @throws IOException + */ + public static void dumpConfiguration(Configuration conf, + Writer out) throws IOException { + Configuration config = new Configuration(conf,true); + config.reloadConfiguration(); + JsonFactory dumpFactory = new JsonFactory(); + JsonGenerator dumpGenerator = dumpFactory.createJsonGenerator(out); + dumpGenerator.writeStartObject(); + dumpGenerator.writeFieldName("properties"); + dumpGenerator.writeStartArray(); + dumpGenerator.flush(); + for (Map.Entry item: config.getProps().entrySet()) { + dumpGenerator.writeStartObject(); + dumpGenerator.writeStringField("key", (String) item.getKey()); + dumpGenerator.writeStringField("value", + config.get((String) item.getKey())); + dumpGenerator.writeBooleanField("isFinal", + config.finalParameters.contains(item.getKey())); + dumpGenerator.writeStringField("resource", + config.updatingResource.get(item.getKey())); + dumpGenerator.writeEndObject(); + } + dumpGenerator.writeEndArray(); + dumpGenerator.writeEndObject(); + dumpGenerator.flush(); + } + /** * Get the {@link ClassLoader} for this job. * diff --git a/src/test/core/org/apache/hadoop/conf/TestConfiguration.java b/src/test/core/org/apache/hadoop/conf/TestConfiguration.java index 52a67b3eb6..12999c3a55 100644 --- a/src/test/core/org/apache/hadoop/conf/TestConfiguration.java +++ b/src/test/core/org/apache/hadoop/conf/TestConfiguration.java @@ -21,12 +21,15 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; +import java.io.StringWriter; import java.util.ArrayList; +import java.util.HashMap; import java.util.Random; import junit.framework.TestCase; import org.apache.hadoop.fs.Path; +import org.codehaus.jackson.map.ObjectMapper; public class TestConfiguration extends TestCase { @@ -408,6 +411,182 @@ public void testClassLoader() { assertTrue(other.getClassLoader() instanceof Fake_ClassLoader); } + static class JsonConfiguration { + JsonProperty[] properties; + + public JsonProperty[] getProperties() { + return properties; + } + + public void setProperties(JsonProperty[] properties) { + this.properties = properties; + } + } + + static class JsonProperty { + String key; + public String getKey() { + return key; + } + public void setKey(String key) { + this.key = key; + } + public String getValue() { + return value; + } + public void setValue(String value) { + this.value = value; + } + public boolean getIsFinal() { + return isFinal; + } + public void setIsFinal(boolean isFinal) { + this.isFinal = isFinal; + } + public String getResource() { + return resource; + } + public void setResource(String resource) { + this.resource = resource; + } + String value; + boolean isFinal; + String resource; + } + + public void testDumpConfiguration () throws IOException { + StringWriter outWriter = new StringWriter(); + Configuration.dumpConfiguration(conf, outWriter); + String jsonStr = outWriter.toString(); + ObjectMapper mapper = new ObjectMapper(); + JsonConfiguration jconf = + mapper.readValue(jsonStr, JsonConfiguration.class); + int defaultLength = jconf.getProperties().length; + + // add 3 keys to the existing configuration properties + out=new BufferedWriter(new FileWriter(CONFIG)); + startConfig(); + appendProperty("test.key1", "value1"); + appendProperty("test.key2", "value2",true); + appendProperty("test.key3", "value3"); + endConfig(); + Path fileResource = new Path(CONFIG); + conf.addResource(fileResource); + out.close(); + + outWriter = new StringWriter(); + Configuration.dumpConfiguration(conf, outWriter); + jsonStr = outWriter.toString(); + mapper = new ObjectMapper(); + jconf = mapper.readValue(jsonStr, JsonConfiguration.class); + int length = jconf.getProperties().length; + // check for consistency in the number of properties parsed in Json format. + assertEquals(length, defaultLength+3); + + //change few keys in another resource file + out=new BufferedWriter(new FileWriter(CONFIG2)); + startConfig(); + appendProperty("test.key1", "newValue1"); + appendProperty("test.key2", "newValue2"); + endConfig(); + Path fileResource1 = new Path(CONFIG2); + conf.addResource(fileResource1); + out.close(); + + outWriter = new StringWriter(); + Configuration.dumpConfiguration(conf, outWriter); + jsonStr = outWriter.toString(); + mapper = new ObjectMapper(); + jconf = mapper.readValue(jsonStr, JsonConfiguration.class); + + // put the keys and their corresponding attributes into a hashmap for their + // efficient retrieval + HashMap confDump = new HashMap(); + for(JsonProperty prop : jconf.getProperties()) { + confDump.put(prop.getKey(), prop); + } + // check if the value and resource of test.key1 is changed + assertEquals("newValue1", confDump.get("test.key1").getValue()); + assertEquals(false, confDump.get("test.key1").getIsFinal()); + assertEquals(fileResource1.toString(), + confDump.get("test.key1").getResource()); + // check if final parameter test.key2 is not changed, since it is first + // loaded as final parameter + assertEquals("value2", confDump.get("test.key2").getValue()); + assertEquals(true, confDump.get("test.key2").getIsFinal()); + assertEquals(fileResource.toString(), + confDump.get("test.key2").getResource()); + // check for other keys which are not modified later + assertEquals("value3", confDump.get("test.key3").getValue()); + assertEquals(false, confDump.get("test.key3").getIsFinal()); + assertEquals(fileResource.toString(), + confDump.get("test.key3").getResource()); + // check for resource to be "Unknown" for keys which are loaded using 'set' + // and expansion of properties + conf.set("test.key4", "value4"); + conf.set("test.key5", "value5"); + conf.set("test.key6", "${test.key5}"); + outWriter = new StringWriter(); + Configuration.dumpConfiguration(conf, outWriter); + jsonStr = outWriter.toString(); + mapper = new ObjectMapper(); + jconf = mapper.readValue(jsonStr, JsonConfiguration.class); + confDump = new HashMap(); + for(JsonProperty prop : jconf.getProperties()) { + confDump.put(prop.getKey(), prop); + } + assertEquals("value5",confDump.get("test.key6").getValue()); + assertEquals("Unknown", confDump.get("test.key4").getResource()); + outWriter.close(); + } + + public void testDumpConfiguratioWithoutDefaults() throws IOException { + // check for case when default resources are not loaded + Configuration config = new Configuration(false); + StringWriter outWriter = new StringWriter(); + Configuration.dumpConfiguration(config, outWriter); + String jsonStr = outWriter.toString(); + ObjectMapper mapper = new ObjectMapper(); + JsonConfiguration jconf = + mapper.readValue(jsonStr, JsonConfiguration.class); + + //ensure that no properties are loaded. + assertEquals(0, jconf.getProperties().length); + + // add 2 keys + out=new BufferedWriter(new FileWriter(CONFIG)); + startConfig(); + appendProperty("test.key1", "value1"); + appendProperty("test.key2", "value2",true); + endConfig(); + Path fileResource = new Path(CONFIG); + config.addResource(fileResource); + out.close(); + + outWriter = new StringWriter(); + Configuration.dumpConfiguration(config, outWriter); + jsonStr = outWriter.toString(); + mapper = new ObjectMapper(); + jconf = mapper.readValue(jsonStr, JsonConfiguration.class); + + HashMapconfDump = new HashMap(); + for (JsonProperty prop : jconf.getProperties()) { + confDump.put(prop.getKey(), prop); + } + //ensure only 2 keys are loaded + assertEquals(2,jconf.getProperties().length); + //ensure the values are consistent + assertEquals(confDump.get("test.key1").getValue(),"value1"); + assertEquals(confDump.get("test.key2").getValue(),"value2"); + //check the final tag + assertEquals(false, confDump.get("test.key1").getIsFinal()); + assertEquals(true, confDump.get("test.key2").getIsFinal()); + //check the resource for each property + for (JsonProperty prop : jconf.getProperties()) { + assertEquals(fileResource.toString(),prop.getResource()); + } + } + public static void main(String[] argv) throws Exception { junit.textui.TestRunner.main(new String[]{ TestConfiguration.class.getName()