big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-17358. Improve excessive reloading of Configurations (#2436)

Browse Source 
Co-authored-by: ahussein <ahmed.hussein@verizonmedia.com>
This commit is contained in:
Ahmed Hussein 2020-11-11 14:39:03 -06:00 committed by GitHub
parent 188ebb5a9b
commit 71071e5c0f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 27 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
View File

@ -1027,11 +1027,11 @@ public synchronized void reloadConfiguration() {
properties = null; // trigger reload
finalParameters.clear(); // clear site-limits
}
private synchronized void addResourceObject(Resource resource) {
resources.add(resource); // add to resources
restrictSystemProps |= resource.isParserRestricted();
reloadConfiguration();
loadProps(properties, resources.size() - 1, false);
}
private static final int MAX_SUBST = 20;
@ -2876,12 +2876,27 @@ public Set<String> getFinalParameters() {
protected synchronized Properties getProps() {
if (properties == null) {
properties = new Properties();
Map<String, String[]> backup = updatingResource != null ?
new ConcurrentHashMap<String, String[]>(updatingResource) : null;
loadResources(properties, resources, quietmode);
loadProps(properties, 0, true);
}
return properties;
}
/**
* Loads the resource at a given index into the properties.
* @param props the object containing the loaded properties.
* @param startIdx the index where the new resource has been added.
* @param fullReload flag whether we do complete reload of the conf instead
* of just loading the new resource.
*/
private synchronized void loadProps(final Properties props,
final int startIdx, final boolean fullReload) {
if (props != null) {
Map<String, String[]> backup =
updatingResource != null
? new ConcurrentHashMap<>(updatingResource) : null;
loadResources(props, resources, startIdx, fullReload, quietmode);
if (overlay != null) {
properties.putAll(overlay);
props.putAll(overlay);
if (backup != null) {
for (Map.Entry<Object, Object> item : overlay.entrySet()) {
String key = (String) item.getKey();
@ -2893,7 +2908,6 @@ protected synchronized Properties getProps() {
}
}
}
return properties;
}
/**
@ -2995,14 +3009,16 @@ private XMLStreamReader parse(InputStream is, String systemIdStr,
private void loadResources(Properties properties,
ArrayList<Resource> resources,
int startIdx,
boolean fullReload,
boolean quiet) {
if(loadDefaults) {
if(loadDefaults && fullReload) {
for (String resource : defaultResources) {
loadResource(properties, new Resource(resource, false), quiet);
}
}
for (int i = 0; i < resources.size(); i++) {
for (int i = startIdx; i < resources.size(); i++) {
Resource ret = loadResource(properties, resources.get(i), quiet);
if (ret != null) {
resources.set(i, ret);

3
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationSubclass.java
View File

@ -53,8 +53,9 @@ public void testReloadNotQuiet() throws Throwable {
SubConf conf = new SubConf(true);
conf.setQuietMode(false);
assertFalse(conf.isReloaded());
// adding a resource does not force a reload.
conf.addResource("not-a-valid-resource");
assertTrue(conf.isReloaded());
assertFalse(conf.isReloaded());
try {
Properties properties = conf.getProperties();
fail("Should not have got here");