big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-15295. Remove redundant logging related to tags from Configuration. Contributed by Ajay Kumar.

Browse Source
This commit is contained in:
Anu Engineer 2018-03-23 21:17:18 -07:00
parent 24f75e097a
commit 28790b81ec
2 changed files with 27 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
View File

@ -229,7 +229,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
private static final Logger LOG_DEPRECATION = private static final Logger LOG_DEPRECATION =
LoggerFactory.getLogger( LoggerFactory.getLogger(
"org.apache.hadoop.conf.Configuration.deprecation"); "org.apache.hadoop.conf.Configuration.deprecation");
private static final Set<String> TAGS = new HashSet<>(); private static final Set<String> TAGS = ConcurrentHashMap.newKeySet();
private boolean quietmode = true; private boolean quietmode = true;
@ -2935,7 +2935,7 @@ private void loadResources(Properties properties,
resources.set(i, ret); resources.set(i, ret);
} }
} }
this.removeUndeclaredTags(properties); this.addTags(properties);
} }
private Resource loadResource(Properties properties, private Resource loadResource(Properties properties,
@ -3183,29 +3183,28 @@ private Resource loadResource(Properties properties,
} }
/** /**
* Removes undeclared tags and related properties from propertyTagsMap. * Add tags defined in HADOOP_SYSTEM_TAGS, HADOOP_CUSTOM_TAGS.
* Its required because ordering of properties in xml config files is not
* guaranteed.
* @param prop * @param prop
*/ */
private void removeUndeclaredTags(Properties prop) { public void addTags(Properties prop) {
// Get all system tags // Get all system tags
if (prop.containsKey(CommonConfigurationKeys.HADOOP_SYSTEM_TAGS)){ try {
String systemTags = prop.getProperty(CommonConfigurationKeys if (prop.containsKey(CommonConfigurationKeys.HADOOP_SYSTEM_TAGS)) {
.HADOOP_SYSTEM_TAGS); String systemTags = prop.getProperty(CommonConfigurationKeys
Arrays.stream(systemTags.split(",")).forEach(tag -> TAGS.add(tag)); .HADOOP_SYSTEM_TAGS);
} Arrays.stream(systemTags.split(",")).forEach(tag -> TAGS.add(tag));
// Get all custom tags }
if (prop.containsKey(CommonConfigurationKeys.HADOOP_CUSTOM_TAGS)) { // Get all custom tags
String customTags = prop.getProperty(CommonConfigurationKeys if (prop.containsKey(CommonConfigurationKeys.HADOOP_CUSTOM_TAGS)) {
.HADOOP_CUSTOM_TAGS); String customTags = prop.getProperty(CommonConfigurationKeys
Arrays.stream(customTags.split(",")).forEach(tag -> TAGS.add(tag)); .HADOOP_CUSTOM_TAGS);
Arrays.stream(customTags.split(",")).forEach(tag -> TAGS.add(tag));
}
} catch (Exception ex) {
LOG.trace("Error adding tags in configuration", ex);
} }
Set undeclaredTags = propertyTagsMap.keySet();
if (undeclaredTags.retainAll(TAGS)) {
LOG.info("Removed undeclared tags:");
}
} }
/** /**
@ -3219,8 +3218,8 @@ private void removeUndeclaredTags(Properties prop) {
private void readTagFromConfig(String attributeValue, String confName, String private void readTagFromConfig(String attributeValue, String confName, String
confValue, List<String> confSource) { confValue, List<String> confSource) {
for (String tagStr : attributeValue.split(",")) { for (String tagStr : attributeValue.split(",")) {
tagStr = tagStr.trim();
try { try {
tagStr = tagStr.trim();
// Handle property with no/null value // Handle property with no/null value
if (confValue == null) { if (confValue == null) {
confValue = ""; confValue = "";

34
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
View File

@ -27,7 +27,6 @@
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.OutputStreamWriter; import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.io.StringWriter; import java.io.StringWriter;
import java.net.InetAddress; import java.net.InetAddress;
import java.net.InetSocketAddress; import java.net.InetSocketAddress;
@ -2392,33 +2391,14 @@ public void testGetAllPropertiesByTags() throws Exception {
@Test @Test
public void testInvalidTags() throws Exception { public void testInvalidTags() throws Exception {
PrintStream output = System.out; Path fileResource = new Path(CONFIG);
try { conf.addResource(fileResource);
ByteArrayOutputStream bytes = new ByteArrayOutputStream(); conf.getProps();
System.setOut(new PrintStream(bytes));
out = new BufferedWriter(new FileWriter(CONFIG)); assertFalse(conf.isPropertyTag("BADTAG"));
startConfig(); assertFalse(conf.isPropertyTag("CUSTOM_TAG"));
appendPropertyByTag("dfs.cblock.trace.io", "false", "MYOWNTAG,TAG2"); assertTrue(conf.isPropertyTag("DEBUG"));
endConfig(); assertTrue(conf.isPropertyTag("HDFS"));
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
conf.getProps();
List<String> tagList = new ArrayList<>();
tagList.add("REQUIRED");
tagList.add("MYOWNTAG");
tagList.add("TAG2");
Properties properties = conf.getAllPropertiesByTags(tagList);
assertEq(0, properties.size());
assertFalse(properties.containsKey("dfs.cblock.trace.io"));
assertFalse(bytes.toString().contains("Invalid tag "));
assertFalse(bytes.toString().contains("Tag"));
} finally {
System.setOut(output);
}
} }
/** /**