big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-15295. Remove redundant logging related to tags from Configuration. Contributed by Ajay Kumar.

Browse Source
This commit is contained in:
Anu Engineer 2018-03-23 21:17:18 -07:00
parent 24f75e097a
commit 28790b81ec
2 changed files with 27 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
View File

@ -229,7 +229,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
private static final Logger LOG_DEPRECATION =
LoggerFactory.getLogger(
"org.apache.hadoop.conf.Configuration.deprecation");
private static final Set<String> TAGS = new HashSet<>();
private static final Set<String> TAGS = ConcurrentHashMap.newKeySet();
private boolean quietmode = true;
@ -2935,7 +2935,7 @@ private void loadResources(Properties properties,
resources.set(i, ret);
}
}
this.removeUndeclaredTags(properties);
this.addTags(properties);
}
private Resource loadResource(Properties properties,
@ -3183,29 +3183,28 @@ private Resource loadResource(Properties properties,
}
/**
* Removes undeclared tags and related properties from propertyTagsMap.
* Its required because ordering of properties in xml config files is not
* guaranteed.
* Add tags defined in HADOOP_SYSTEM_TAGS, HADOOP_CUSTOM_TAGS.
* @param prop
*/
private void removeUndeclaredTags(Properties prop) {
public void addTags(Properties prop) {
// Get all system tags
if (prop.containsKey(CommonConfigurationKeys.HADOOP_SYSTEM_TAGS)){
String systemTags = prop.getProperty(CommonConfigurationKeys
.HADOOP_SYSTEM_TAGS);
Arrays.stream(systemTags.split(",")).forEach(tag -> TAGS.add(tag));
}
// Get all custom tags
if (prop.containsKey(CommonConfigurationKeys.HADOOP_CUSTOM_TAGS)) {
String customTags = prop.getProperty(CommonConfigurationKeys
.HADOOP_CUSTOM_TAGS);
Arrays.stream(customTags.split(",")).forEach(tag -> TAGS.add(tag));
try {
if (prop.containsKey(CommonConfigurationKeys.HADOOP_SYSTEM_TAGS)) {
String systemTags = prop.getProperty(CommonConfigurationKeys
.HADOOP_SYSTEM_TAGS);
Arrays.stream(systemTags.split(",")).forEach(tag -> TAGS.add(tag));
}
// Get all custom tags
if (prop.containsKey(CommonConfigurationKeys.HADOOP_CUSTOM_TAGS)) {
String customTags = prop.getProperty(CommonConfigurationKeys
.HADOOP_CUSTOM_TAGS);
Arrays.stream(customTags.split(",")).forEach(tag -> TAGS.add(tag));
}
} catch (Exception ex) {
LOG.trace("Error adding tags in configuration", ex);
}
Set undeclaredTags = propertyTagsMap.keySet();
if (undeclaredTags.retainAll(TAGS)) {
LOG.info("Removed undeclared tags:");
}
}
/**
@ -3219,8 +3218,8 @@ private void removeUndeclaredTags(Properties prop) {
private void readTagFromConfig(String attributeValue, String confName, String
confValue, List<String> confSource) {
for (String tagStr : attributeValue.split(",")) {
tagStr = tagStr.trim();
try {
tagStr = tagStr.trim();
// Handle property with no/null value
if (confValue == null) {
confValue = "";

34
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
View File

@ -27,7 +27,6 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.io.StringWriter;
import java.net.InetAddress;
import java.net.InetSocketAddress;
@ -2392,33 +2391,14 @@ public void testGetAllPropertiesByTags() throws Exception {
@Test
public void testInvalidTags() throws Exception {
PrintStream output = System.out;
try {
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
System.setOut(new PrintStream(bytes));
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
conf.getProps();
out = new BufferedWriter(new FileWriter(CONFIG));
startConfig();
appendPropertyByTag("dfs.cblock.trace.io", "false", "MYOWNTAG,TAG2");
endConfig();
Path fileResource = new Path(CONFIG);
conf.addResource(fileResource);
conf.getProps();
List<String> tagList = new ArrayList<>();
tagList.add("REQUIRED");
tagList.add("MYOWNTAG");
tagList.add("TAG2");
Properties properties = conf.getAllPropertiesByTags(tagList);
assertEq(0, properties.size());
assertFalse(properties.containsKey("dfs.cblock.trace.io"));
assertFalse(bytes.toString().contains("Invalid tag "));
assertFalse(bytes.toString().contains("Tag"));
} finally {
System.setOut(output);
}
assertFalse(conf.isPropertyTag("BADTAG"));
assertFalse(conf.isPropertyTag("CUSTOM_TAG"));
assertTrue(conf.isPropertyTag("DEBUG"));
assertTrue(conf.isPropertyTag("HDFS"));
}
/**