big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-6184. Provide an API to dump Configuration in a JSON format. Contributed by V.V.Chaitanya Krishna.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@807149 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Hemanth Yamijala 2009-08-24 11:21:18 +00:00
parent 76a77aea78
commit 280ab0cf7d
3 changed files with 262 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGES.txt
View File

@ -504,6 +504,9 @@ Trunk (unreleased changes)
HADOOP-6173. Change src/native/packageNativeHadoop.sh to package all
native library files. (Hong Tang via szetszwo)
HADOOP-6184. Provide an API to dump Configuration in a JSON format.
(V.V.Chaitanya Krishna via yhemanth)
OPTIMIZATIONS

82
src/java/org/apache/hadoop/conf/Configuration.java
View File

@ -28,6 +28,7 @@
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.Reader;
import java.io.Writer;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collection;
@ -62,6 +63,8 @@
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.JsonGenerator;
import org.w3c.dom.DOMException;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
@ -155,7 +158,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
private boolean loadDefaults = true;
/**
* Configurtion objects
* Configuration objects
*/
private static final WeakHashMap<Configuration,Object> REGISTRY =
new WeakHashMap<Configuration,Object>();
@ -167,6 +170,18 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
private static final ArrayList<String> defaultResources =
new ArrayList<String>();
/**
* Flag to indicate if the storage of resource which updates a key needs
* to be stored for each key
*/
private boolean storeResource;
/**
* Stores the mapping of key to the resource which modifies or loads
* the key most recently
*/
private HashMap<String, String> updatingResource;
static{
//print deprecation warning if hadoop-site.xml is found in classpath
ClassLoader cL = Thread.currentThread().getContextClassLoader();
@ -214,6 +229,23 @@ public Configuration(boolean loadDefaults) {
synchronized(Configuration.class) {
REGISTRY.put(this, null);
}
this.storeResource = false;
}
/**
* A new configuration with the same settings and additional facility for
* storage of resource to each key which loads or updates
* the key most recently
* @param other the configuration from which to clone settings
* @param storeResource flag to indicate if the storage of resource to
* each key is to be stored
*/
private Configuration(Configuration other, boolean storeResource) {
this(other);
this.storeResource = storeResource;
if (storeResource) {
updatingResource = new HashMap<String, String>();
}
}
/**
@ -1081,8 +1113,14 @@ protected synchronized Properties getProps() {
if (properties == null) {
properties = new Properties();
loadResources(properties, resources, quietmode);
if (overlay!= null)
if (overlay!= null) {
properties.putAll(overlay);
if (storeResource) {
for (Map.Entry<Object,Object> item: overlay.entrySet()) {
updatingResource.put((String) item.getKey(), "Unknown");
}
}
}
}
return properties;
}
@ -1251,6 +1289,9 @@ private void loadResource(Properties properties, Object name, boolean quiet) {
if (attr != null && value != null) {
if (!finalParameters.contains(attr)) {
properties.setProperty(attr, value);
if (storeResource) {
updatingResource.put(attr, name.toString());
}
if (finalParameter)
finalParameters.add(attr);
} else {
@ -1322,6 +1363,43 @@ public void writeXml(OutputStream out) throws IOException {
}
}
/**
* Writes out all the parameters and their properties (final and resource) to
* the given {@link Writer}
* The format of the output would be
* { "properties" : [ {key1,value1,key1.isFinal,key1.resource}, {key2,value2,
* key2.isFinal,key2.resource}... ] }
* It does not output the parameters of the configuration object which is
* loaded from an input stream.
* @param out the Writer to write to
* @throws IOException
*/
public static void dumpConfiguration(Configuration conf,
Writer out) throws IOException {
Configuration config = new Configuration(conf,true);
config.reloadConfiguration();
JsonFactory dumpFactory = new JsonFactory();
JsonGenerator dumpGenerator = dumpFactory.createJsonGenerator(out);
dumpGenerator.writeStartObject();
dumpGenerator.writeFieldName("properties");
dumpGenerator.writeStartArray();
dumpGenerator.flush();
for (Map.Entry<Object,Object> item: config.getProps().entrySet()) {
dumpGenerator.writeStartObject();
dumpGenerator.writeStringField("key", (String) item.getKey());
dumpGenerator.writeStringField("value",
config.get((String) item.getKey()));
dumpGenerator.writeBooleanField("isFinal",
config.finalParameters.contains(item.getKey()));
dumpGenerator.writeStringField("resource",
config.updatingResource.get(item.getKey()));
dumpGenerator.writeEndObject();
}
dumpGenerator.writeEndArray();
dumpGenerator.writeEndObject();
dumpGenerator.flush();
}
/**
* Get the {@link ClassLoader} for this job.
*

179
src/test/core/org/apache/hadoop/conf/TestConfiguration.java
View File

@ -21,12 +21,15 @@
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Random;
import junit.framework.TestCase;
import org.apache.hadoop.fs.Path;
import org.codehaus.jackson.map.ObjectMapper;
public class TestConfiguration extends TestCase {
@ -408,6 +411,182 @@ public void testClassLoader() {
assertTrue(other.getClassLoader() instanceof Fake_ClassLoader);
}
static class JsonConfiguration {
JsonProperty[] properties;
public JsonProperty[] getProperties() {
return properties;
}
public void setProperties(JsonProperty[] properties) {
this.properties = properties;
}
}
static class JsonProperty {
String key;
public String getKey() {
return key;
}
public void setKey(String key) {
this.key = key;
}
public String getValue() {
return value;
}
public void setValue(String value) {
this.value = value;
}
public boolean getIsFinal() {
return isFinal;
}
public void setIsFinal(boolean isFinal) {
this.isFinal = isFinal;
}
public String getResource() {
return resource;
}
public void setResource(String resource) {
this.resource = resource;
}
String value;
boolean isFinal;
String resource;
}
public void testDumpConfiguration () throws IOException {
StringWriter outWriter = new StringWriter();
Configuration.dumpConfiguration(conf, outWriter);
String jsonStr = outWriter.toString();
ObjectMapper mapper = new ObjectMapper();
JsonConfiguration jconf =
mapper.readValue(jsonStr, JsonConfiguration.class);
int defaultLength = jconf.getProperties().length;
// add 3 keys to the existing configuration properties
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("test.key1", "value1");
appendProperty("test.key2", "value2",true);
appendProperty("test.key3", "value3");
endConfig();
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
out.close();
outWriter = new StringWriter();
Configuration.dumpConfiguration(conf, outWriter);
jsonStr = outWriter.toString();
mapper = new ObjectMapper();
jconf = mapper.readValue(jsonStr, JsonConfiguration.class);
int length = jconf.getProperties().length;
// check for consistency in the number of properties parsed in Json format.
assertEquals(length, defaultLength+3);
//change few keys in another resource file
out=new BufferedWriter(new FileWriter(CONFIG2));
startConfig();
appendProperty("test.key1", "newValue1");
appendProperty("test.key2", "newValue2");
endConfig();
Path fileResource1 = new Path(CONFIG2);
conf.addResource(fileResource1);
out.close();
outWriter = new StringWriter();
Configuration.dumpConfiguration(conf, outWriter);
jsonStr = outWriter.toString();
mapper = new ObjectMapper();
jconf = mapper.readValue(jsonStr, JsonConfiguration.class);
// put the keys and their corresponding attributes into a hashmap for their
// efficient retrieval
HashMap<String,JsonProperty> confDump = new HashMap<String,JsonProperty>();
for(JsonProperty prop : jconf.getProperties()) {
confDump.put(prop.getKey(), prop);
}
// check if the value and resource of test.key1 is changed
assertEquals("newValue1", confDump.get("test.key1").getValue());
assertEquals(false, confDump.get("test.key1").getIsFinal());
assertEquals(fileResource1.toString(),
confDump.get("test.key1").getResource());
// check if final parameter test.key2 is not changed, since it is first
// loaded as final parameter
assertEquals("value2", confDump.get("test.key2").getValue());
assertEquals(true, confDump.get("test.key2").getIsFinal());
assertEquals(fileResource.toString(),
confDump.get("test.key2").getResource());
// check for other keys which are not modified later
assertEquals("value3", confDump.get("test.key3").getValue());
assertEquals(false, confDump.get("test.key3").getIsFinal());
assertEquals(fileResource.toString(),
confDump.get("test.key3").getResource());
// check for resource to be "Unknown" for keys which are loaded using 'set'
// and expansion of properties
conf.set("test.key4", "value4");
conf.set("test.key5", "value5");
conf.set("test.key6", "${test.key5}");
outWriter = new StringWriter();
Configuration.dumpConfiguration(conf, outWriter);
jsonStr = outWriter.toString();
mapper = new ObjectMapper();
jconf = mapper.readValue(jsonStr, JsonConfiguration.class);
confDump = new HashMap<String, JsonProperty>();
for(JsonProperty prop : jconf.getProperties()) {
confDump.put(prop.getKey(), prop);
}
assertEquals("value5",confDump.get("test.key6").getValue());
assertEquals("Unknown", confDump.get("test.key4").getResource());
outWriter.close();
}
public void testDumpConfiguratioWithoutDefaults() throws IOException {
// check for case when default resources are not loaded
Configuration config = new Configuration(false);
StringWriter outWriter = new StringWriter();
Configuration.dumpConfiguration(config, outWriter);
String jsonStr = outWriter.toString();
ObjectMapper mapper = new ObjectMapper();
JsonConfiguration jconf =
mapper.readValue(jsonStr, JsonConfiguration.class);
//ensure that no properties are loaded.
assertEquals(0, jconf.getProperties().length);
// add 2 keys
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendProperty("test.key1", "value1");
appendProperty("test.key2", "value2",true);
endConfig();
Path fileResource = new Path(CONFIG);
config.addResource(fileResource);
out.close();
outWriter = new StringWriter();
Configuration.dumpConfiguration(config, outWriter);
jsonStr = outWriter.toString();
mapper = new ObjectMapper();
jconf = mapper.readValue(jsonStr, JsonConfiguration.class);
HashMap<String, JsonProperty>confDump = new HashMap<String, JsonProperty>();
for (JsonProperty prop : jconf.getProperties()) {
confDump.put(prop.getKey(), prop);
}
//ensure only 2 keys are loaded
assertEquals(2,jconf.getProperties().length);
//ensure the values are consistent
assertEquals(confDump.get("test.key1").getValue(),"value1");
assertEquals(confDump.get("test.key2").getValue(),"value2");
//check the final tag
assertEquals(false, confDump.get("test.key1").getIsFinal());
assertEquals(true, confDump.get("test.key2").getIsFinal());
//check the resource for each property
for (JsonProperty prop : jconf.getProperties()) {
assertEquals(fileResource.toString(),prop.getResource());
}
}
public static void main(String[] argv) throws Exception {
junit.textui.TestRunner.main(new String[]{
TestConfiguration.class.getName()