diff --git a/CHANGES.txt b/CHANGES.txt index 834506aae5..450195dfd0 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -94,6 +94,9 @@ Trunk (unreleased changes) HADOOP-6479. TestUTF8 assertions could fail with better text. (Steve Loughran via tomwhite) + HADOOP-6420. Add functionality permitting subsets of Configuration to be + interpreted as Map. (Aaron Kimball via cdouglas) + OPTIMIZATIONS BUG FIXES diff --git a/src/java/org/apache/hadoop/conf/Configuration.java b/src/java/org/apache/hadoop/conf/Configuration.java index 7407d8b942..e75c709400 100644 --- a/src/java/org/apache/hadoop/conf/Configuration.java +++ b/src/java/org/apache/hadoop/conf/Configuration.java @@ -31,6 +31,7 @@ import java.io.Reader; import java.io.Writer; import java.net.URL; +import java.util.AbstractMap; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -1067,7 +1068,139 @@ public String[] getTrimmedStrings(String name, String... defaultValue) { public void setStrings(String name, String... values) { set(name, StringUtils.arrayToString(values)); } - + + /** + * Instantiates a map view over a subset of the entries in + * the Configuration. This is instantiated by getMap(), which + * binds a prefix of the namespace to the ConfigItemMap. This + * mapping reflects changes to the underlying Configuration. + * + * This map does not support iteration. + */ + protected class ConfigItemMap extends AbstractMap + implements Map { + + private final String prefix; + + public ConfigItemMap(String prefix) { + this.prefix = prefix; + } + + @Override + public boolean containsKey(Object key) { + return lookup(key.toString()) != null; + } + + @Override + public Set> entrySet() { + throw new UnsupportedOperationException("unsupported"); + } + + @Override + public boolean equals(Object o) { + return o != null && o instanceof ConfigItemMap + && prefix.equals(((ConfigItemMap) o).prefix) + && Configuration.this == ((ConfigItemMap) o).getConfiguration(); + } + + private Configuration getConfiguration() { + return Configuration.this; + } + + @Override + public String get(Object key) { + if (null == key) { + return null; + } + + return lookup(key.toString()); + } + + @Override + public int hashCode() { + return prefix.hashCode(); + } + + @Override + public String put(String key, String val) { + if (null == key) { + return null; + } + + String ret = get(key); + Configuration.this.set(prefix + key, val); + return ret; + } + + @Override + public void putAll(Map m) { + for (Map.Entry entry : m.entrySet()) { + put(entry.getKey(), entry.getValue()); + } + } + + private String lookup(String subKey) { + String configKey = prefix + subKey; + Properties props = Configuration.this.getProps(); + Object val = props.get(configKey); + String str = null; + if (null != val) { + str = substituteVars(val.toString()); + } + + return str; + } + } + + /** + * Given a string -> string map as a value, embed this in the + * Configuration by prepending 'name' to all the keys in the valueMap, + * and storing it inside the current Configuration. + * + * e.g., setMap("foo", { "bar" -> "a", "baz" -> "b" }) would + * insert "foo.bar" -> "a" and "foo.baz" -> "b" in this + * Configuration. + * + * @param name the prefix to attach to all keys in the valueMap. This + * should not have a trailing "." character. + * @param valueMap the map to embed in the Configuration. + */ + public void setMap(String name, Map valueMap) { + // Store all elements of the map proper. + for (Map.Entry entry : valueMap.entrySet()) { + set(name + "." + entry.getKey(), entry.getValue()); + } + } + + /** + * Returns a map containing a view of all configuration properties + * whose names begin with "name.*", with the "name." prefix removed. + * e.g., if "foo.bar" -> "a" and "foo.baz" -> "b" are in the + * Configuration, getMap("foo") would return { "bar" -> "a", + * "baz" -> "b" }. + * + * Map name deprecation is handled via "prefix deprecation"; the individual + * keys created in a configuration by inserting a map do not need to be + * individually deprecated -- it is sufficient to deprecate the 'name' + * associated with the map and bind that to a new name. e.g., if "foo" + * is deprecated for "newfoo," and the configuration contains entries for + * "newfoo.a" and "newfoo.b", getMap("foo") will return a map containing + * the keys "a" and "b". + * + * The returned map does not support iteration; it is a lazy view over + * the slice of the configuration whose keys begin with 'name'. Updates + * to the underlying configuration are reflected in the returned map, + * and updates to the map will modify the underlying configuration. + * + * @param name The prefix of the key names to extract into the output map. + * @return a String->String map that contains all (k, v) pairs + * where 'k' begins with 'name.'; the 'name.' prefix is removed in the output. + */ + public Map getMap(String name) { + String prefix = handleDeprecation(name) + "."; + return new ConfigItemMap(prefix); + } + /** * Load a class by name. * diff --git a/src/test/core/org/apache/hadoop/conf/TestConfiguration.java b/src/test/core/org/apache/hadoop/conf/TestConfiguration.java index 6c70d34dd2..c77136c7dd 100644 --- a/src/test/core/org/apache/hadoop/conf/TestConfiguration.java +++ b/src/test/core/org/apache/hadoop/conf/TestConfiguration.java @@ -24,6 +24,7 @@ import java.io.StringWriter; import java.util.ArrayList; import java.util.HashMap; +import java.util.Map; import java.util.Random; import java.util.regex.Pattern; @@ -365,6 +366,49 @@ public void testEnum() throws IOException { assertTrue(fail); } + public void testMap() throws IOException { + Configuration conf = new Configuration(); + + // manually create a map in the config; extract + // its values as a map object. + conf.set("foo.bar", "A"); + conf.set("foo.baz", "B"); + assertEquals("A", conf.get("foo.bar")); + assertEquals("B", conf.get("foo.baz")); + + Map out = conf.getMap("foo"); + assertEquals("A", out.get("bar")); + assertEquals("B", out.get("baz")); + + Map in = new HashMap(); + in.put("yak", "123"); + in.put("bop", "456"); + conf.setMap("quux", in); + + // Assert that we can extract individual entries in + // the nested map ok. + assertEquals("123", conf.get("quux.yak")); + + // Assert that we can get the whole map back out again. + out = conf.getMap("quux"); + assertEquals("123", out.get("yak")); + assertEquals("456", out.get("bop")); + + // Test that substitution is handled by getMap(). + conf.set("subparam", "foo"); + conf.set("mymap.someprop", "AAA${subparam}BBB"); + out = conf.getMap("mymap"); + assertEquals("AAAfooBBB", out.get("someprop")); + + // Test deprecation of maps. + Configuration.addDeprecation("oldfoo", new String[]{"newfoo"}); + conf.set("newfoo.a", "A"); + conf.set("newfoo.b", "B"); + out = conf.getMap("oldfoo"); + assertEquals("A", out.get("a")); + assertEquals("B", out.get("b")); + } + public void testPattern() throws IOException { out = new BufferedWriter(new FileWriter(CONFIG)); startConfig();