diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index a1a40b9701..a339dac6fe 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -259,7 +259,18 @@ public String toString() { */ private static final WeakHashMap REGISTRY = new WeakHashMap(); - + + /** + * Map to register all classes holding property tag enums. + */ + private static final Map + REGISTERED_TAG_CLASS = new HashMap<>(); + /** + * Map to hold properties by there tag groupings. + */ + private final Map propertyTagsMap = + new ConcurrentHashMap<>(); + /** * List of default Resources. Resources are loaded in the order of the list * entries @@ -738,6 +749,12 @@ public Configuration() { public Configuration(boolean loadDefaults) { this.loadDefaults = loadDefaults; updatingResource = new ConcurrentHashMap(); + + // Register all classes holding property tags with + REGISTERED_TAG_CLASS.put("core", CorePropertyTag.class); + REGISTERED_TAG_CLASS.put("hdfs", HDFSPropertyTag.class); + REGISTERED_TAG_CLASS.put("yarn", YarnPropertyTag.class); + synchronized(Configuration.class) { REGISTRY.put(this, null); } @@ -765,6 +782,8 @@ public Configuration(Configuration other) { this.finalParameters = Collections.newSetFromMap( new ConcurrentHashMap()); this.finalParameters.addAll(other.finalParameters); + this.REGISTERED_TAG_CLASS.putAll(other.REGISTERED_TAG_CLASS); + this.propertyTagsMap.putAll(other.propertyTagsMap); } synchronized(Configuration.class) { @@ -2823,6 +2842,10 @@ private Resource loadResource(Properties properties, } else if ("source".equals(propertyAttr)) { confSource.add(StringInterner.weakIntern( reader.getAttributeValue(i))); + } else if ("tag".equals(propertyAttr)) { + //Read tags and put them in propertyTagsMap + readTagFromConfig(reader.getAttributeValue(i), confName, + confValue, confSource); } } break; @@ -2830,6 +2853,7 @@ private Resource loadResource(Properties properties, case "value": case "final": case "source": + case "tag": parseToken = true; token.setLength(0); break; @@ -2911,6 +2935,13 @@ private Resource loadResource(Properties properties, case "source": confSource.add(StringInterner.weakIntern(token.toString())); break; + case "tag": + if (token.length() > 0) { + //Read tags and put them in propertyTagsMap + readTagFromConfig(token.toString(), confName, + confValue, confSource); + } + break; case "include": if (fallbackAllowed && !fallbackEntered) { throw new IOException("Fetch fail on include for '" @@ -2962,6 +2993,48 @@ private Resource loadResource(Properties properties, } } + private void readTagFromConfig(String attributeValue, String confName, String + confValue, List confSource) { + for (String tagStr : attributeValue.split(",")) { + tagStr = tagStr.trim(); + try { + if (confSource.size() > 0) { + for (String source : confSource) { + PropertyTag tag1 = this.getPropertyTag(tagStr, + source.split("-")[0]); + if (propertyTagsMap.containsKey(tag1)) { + propertyTagsMap.get(tag1) + .setProperty(confName, confValue); + } else { + Properties props = new Properties(); + props.setProperty(confName, confValue); + propertyTagsMap.put(tag1, props); + } + } + } else { + //If no source is set try to find tag in CorePropertyTag + if (propertyTagsMap + .containsKey(CorePropertyTag.valueOf(tagStr) + )) { + propertyTagsMap.get(CorePropertyTag.valueOf(tagStr)) + .setProperty(confName, confValue); + } else { + Properties props = new Properties(); + props.setProperty(confName, confValue); + propertyTagsMap.put(CorePropertyTag.valueOf(tagStr), + props); + } + } + } catch (IllegalArgumentException iae) { + //Log the invalid tag and continue to parse rest of the + // properties. + LOG.info("Invalid tag '" + tagStr + "' found for " + + "property:" + confName, iae); + } + + } + } + private void overlay(Properties to, Properties from) { for (Entry entry: from.entrySet()) { to.put(entry.getKey(), entry.getValue()); @@ -3438,4 +3511,45 @@ public static boolean hasWarnedDeprecation(String name) { } return false; } + + /** + * Get all properties belonging to tag. + * @return Properties with matching properties + */ + public Properties getAllPropertiesByTag(final PropertyTag tag) { + Properties props = new Properties(); + if (propertyTagsMap.containsKey(tag)) { + props.putAll(propertyTagsMap.get(tag)); + } + return props; + } + + /** + * Get all properties belonging to list of input tags. Calls + * getAllPropertiesByTag internally. + * + * @return Properties with all matching properties + */ + public Properties getAllPropertiesByTags(final List tagList) { + Properties prop = new Properties(); + for (PropertyTag tag : tagList) { + prop.putAll(this.getAllPropertiesByTag(tag)); + } + return prop; + } + + /** + * Get Property tag Enum corresponding to given source. + * + * @param tagStr String representation of Enum + * @param group Group to which enum belongs.Ex hdfs,yarn + * @return Properties with all matching properties + */ + private PropertyTag getPropertyTag(String tagStr, String group) { + PropertyTag tag = null; + if (REGISTERED_TAG_CLASS.containsKey(group)) { + tag = (PropertyTag) Enum.valueOf(REGISTERED_TAG_CLASS.get(group), tagStr); + } + return tag; + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/CorePropertyTag.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/CorePropertyTag.java new file mode 100644 index 0000000000..54a75b855b --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/CorePropertyTag.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.conf; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/*************************************************************************** + * Enum for tagging hadoop core properties according to there usage. + * CorePropertyTag implements the + * {@link org.apache.hadoop.conf.PropertyTag} interface, + ***************************************************************************/ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public enum CorePropertyTag implements PropertyTag { + CORE, + REQUIRED, + PERFORMANCE, + CLIENT, + SERVER, + SECURITY, + DEBUG +} \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/HDFSPropertyTag.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/HDFSPropertyTag.java new file mode 100644 index 0000000000..02dfb869ae --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/HDFSPropertyTag.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.conf; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/*************************************************************************** + * Enum for tagging hdfs properties according to there usage or application. + * HDFSPropertyTag implements the + * {@link org.apache.hadoop.conf.PropertyTag} interface, + ***************************************************************************/ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public enum HDFSPropertyTag implements PropertyTag { + HDFS, + NAMENODE, + DATANODE, + REQUIRED, + SECURITY, + KERBEROS, + PERFORMANCE, + CLIENT, + SERVER, + DEBUG, + DEPRICATED +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/PropertyTag.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/PropertyTag.java new file mode 100644 index 0000000000..df8d4f9665 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/PropertyTag.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.conf; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/*********************************************************** + * PropertyTag is used for creating extendable property tag Enums. + * Property tags will group related properties together. + ***********************************************************/ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public interface PropertyTag { + +} \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/YarnPropertyTag.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/YarnPropertyTag.java new file mode 100644 index 0000000000..e7a9c797d8 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/YarnPropertyTag.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.conf; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/*************************************************************************** + * Enum for tagging yarn properties according to there usage or application. + * YarnPropertyTag implements the + * {@link org.apache.hadoop.conf.PropertyTag} interface, + ***************************************************************************/ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public enum YarnPropertyTag implements PropertyTag { + YARN, + RESOURCEMANAGER, + SECURITY, + KERBEROS, + PERFORMANCE, + CLIENT, + REQUIRED, + SERVER, + DEBUG +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java index b41a8074d0..4cd1666eca 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java @@ -38,6 +38,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.Random; import java.util.Set; import java.util.regex.Pattern; @@ -53,7 +54,6 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration.IntegerRanges; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; @@ -135,6 +135,7 @@ private void writeConfiguration() throws IOException{ private void endConfig() throws IOException{ out.write("\n"); + out.flush(); out.close(); } @@ -577,6 +578,34 @@ void appendProperty(String name, String val, boolean isFinal, out.write("\n"); } + private void appendPropertyByTag(String name, String val, String tags, + String... sources) throws IOException { + appendPropertyByTag(name, val, false, tags, sources); + } + + private void appendPropertyByTag(String name, String val, boolean isFinal, + String tag, String... sources) throws IOException { + out.write(""); + out.write(""); + out.write(name); + out.write(""); + out.write(""); + out.write(val); + out.write(""); + if (isFinal) { + out.write("true"); + } + for (String s : sources) { + out.write(""); + out.write(s); + out.write(""); + } + out.write(""); + out.write(tag); + out.write(""); + out.write("\n"); + } + void appendCompactFormatProperty(String name, String val) throws IOException { appendCompactFormatProperty(name, val, false); } @@ -2215,4 +2244,85 @@ public static void main(String[] argv) throws Exception { TestConfiguration.class.getName() }); } + + @Test + public void testGetAllPropertiesByTags() throws Exception { + + out = new BufferedWriter(new FileWriter(CONFIG)); + startConfig(); + appendPropertyByTag("dfs.cblock.trace.io", "false", "DEBUG"); + appendPropertyByTag("dfs.replication", "1", "PERFORMANCE,REQUIRED"); + appendPropertyByTag("dfs.namenode.logging.level", "INFO", "CLIENT,DEBUG"); + endConfig(); + + Path fileResource = new Path(CONFIG); + conf.addResource(fileResource); + conf.getProps(); + + List tagList = new ArrayList<>(); + tagList.add(CorePropertyTag.REQUIRED); + tagList.add(CorePropertyTag.PERFORMANCE); + tagList.add(CorePropertyTag.DEBUG); + tagList.add(CorePropertyTag.CLIENT); + + Properties properties = conf.getAllPropertiesByTags(tagList); + assertEq(3, properties.size()); + assertEq(true, properties.containsKey("dfs.namenode.logging.level")); + assertEq(true, properties.containsKey("dfs.replication")); + assertEq(true, properties.containsKey("dfs.cblock.trace.io")); + assertEq(false, properties.containsKey("namenode.host")); + } + + @Test + public void testGetAllPropertiesWithSourceByTags() throws Exception { + + out = new BufferedWriter(new FileWriter(CONFIG)); + startConfig(); + appendPropertyByTag("dfs.cblock.trace.io", "false", "DEBUG", + "hdfs-default.xml", "core-site.xml"); + appendPropertyByTag("dfs.replication", "1", "PERFORMANCE,HDFS", + "hdfs-default.xml"); + appendPropertyByTag("yarn.resourcemanager.work-preserving-recovery" + + ".enabled", "INFO", "CLIENT,DEBUG", "yarn-default.xml", "yarn-site" + + ".xml"); + endConfig(); + + Path fileResource = new Path(CONFIG); + conf.addResource(fileResource); + conf.getProps(); + + List tagList = new ArrayList<>(); + tagList.add(CorePropertyTag.REQUIRED); + + Properties properties; + properties = conf.getAllPropertiesByTags(tagList); + assertNotEquals(3, properties.size()); + + tagList.add(HDFSPropertyTag.DEBUG); + tagList.add(YarnPropertyTag.CLIENT); + tagList.add(HDFSPropertyTag.PERFORMANCE); + tagList.add(HDFSPropertyTag.HDFS); + properties = conf.getAllPropertiesByTags(tagList); + assertEq(3, properties.size()); + + assertEq(true, properties.containsKey("dfs.cblock.trace.io")); + assertEq(true, properties.containsKey("dfs.replication")); + assertEq(true, properties + .containsKey("yarn.resourcemanager.work-preserving-recovery.enabled")); + assertEq(false, properties.containsKey("namenode.host")); + + tagList.clear(); + tagList.add(HDFSPropertyTag.DEBUG); + properties = conf.getAllPropertiesByTags(tagList); + assertEq(true, properties.containsKey("dfs.cblock.trace.io")); + assertEq(false, properties.containsKey("yarn.resourcemanager" + + ".work-preserving-recovery")); + + tagList.clear(); + tagList.add(YarnPropertyTag.DEBUG); + properties = conf.getAllPropertiesByTags(tagList); + assertEq(false, properties.containsKey("dfs.cblock.trace.io")); + assertEq(true, properties.containsKey("yarn.resourcemanager" + + ".work-preserving-recovery.enabled")); + } }