big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-17358. Improve excessive reloading of Configurations (#2436)

Browse Source 
Co-authored-by: ahussein <ahmed.hussein@verizonmedia.com>
(cherry picked from commit 71071e5c0f)
This commit is contained in:
Ahmed Hussein 2020-11-11 14:39:03 -06:00 committed by Wei-Chiu Chuang
parent 47131cdf7c
commit 23fe3bdab3
2 changed files with 27 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
View File

@ -1027,11 +1027,11 @@ public synchronized void reloadConfiguration() {
properties = null; // trigger reload properties = null; // trigger reload
finalParameters.clear(); // clear site-limits finalParameters.clear(); // clear site-limits
} }
private synchronized void addResourceObject(Resource resource) { private synchronized void addResourceObject(Resource resource) {
resources.add(resource); // add to resources resources.add(resource); // add to resources
restrictSystemProps |= resource.isParserRestricted(); restrictSystemProps |= resource.isParserRestricted();
reloadConfiguration(); loadProps(properties, resources.size() - 1, false);
} }
private static final int MAX_SUBST = 20; private static final int MAX_SUBST = 20;
@ -2872,12 +2872,27 @@ public Set<String> getFinalParameters() {
protected synchronized Properties getProps() { protected synchronized Properties getProps() {
if (properties == null) { if (properties == null) {
properties = new Properties(); properties = new Properties();
Map<String, String[]> backup = updatingResource != null ? loadProps(properties, 0, true);
new ConcurrentHashMap<String, String[]>(updatingResource) : null; }
loadResources(properties, resources, quietmode); return properties;
}
/**
* Loads the resource at a given index into the properties.
* @param props the object containing the loaded properties.
* @param startIdx the index where the new resource has been added.
* @param fullReload flag whether we do complete reload of the conf instead
* of just loading the new resource.
*/
private synchronized void loadProps(final Properties props,
final int startIdx, final boolean fullReload) {
if (props != null) {
Map<String, String[]> backup =
updatingResource != null
? new ConcurrentHashMap<>(updatingResource) : null;
loadResources(props, resources, startIdx, fullReload, quietmode);
if (overlay != null) { if (overlay != null) {
properties.putAll(overlay); props.putAll(overlay);
if (backup != null) { if (backup != null) {
for (Map.Entry<Object, Object> item : overlay.entrySet()) { for (Map.Entry<Object, Object> item : overlay.entrySet()) {
String key = (String) item.getKey(); String key = (String) item.getKey();
@ -2889,7 +2904,6 @@ protected synchronized Properties getProps() {
} }
} }
} }
return properties;
} }
/** /**
@ -2991,14 +3005,16 @@ private XMLStreamReader parse(InputStream is, String systemIdStr,
private void loadResources(Properties properties, private void loadResources(Properties properties,
ArrayList<Resource> resources, ArrayList<Resource> resources,
int startIdx,
boolean fullReload,
boolean quiet) { boolean quiet) {
if(loadDefaults) { if(loadDefaults && fullReload) {
for (String resource : defaultResources) { for (String resource : defaultResources) {
loadResource(properties, new Resource(resource, false), quiet); loadResource(properties, new Resource(resource, false), quiet);
} }
} }
for (int i = 0; i < resources.size(); i++) { for (int i = startIdx; i < resources.size(); i++) {
Resource ret = loadResource(properties, resources.get(i), quiet); Resource ret = loadResource(properties, resources.get(i), quiet);
if (ret != null) { if (ret != null) {
resources.set(i, ret); resources.set(i, ret);

3
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationSubclass.java
View File

@ -53,8 +53,9 @@ public void testReloadNotQuiet() throws Throwable {
SubConf conf = new SubConf(true); SubConf conf = new SubConf(true);
conf.setQuietMode(false); conf.setQuietMode(false);
assertFalse(conf.isReloaded()); assertFalse(conf.isReloaded());
// adding a resource does not force a reload.
conf.addResource("not-a-valid-resource"); conf.addResource("not-a-valid-resource");
assertTrue(conf.isReloaded()); assertFalse(conf.isReloaded());
try { try {
Properties properties = conf.getProperties(); Properties properties = conf.getProperties();
fail("Should not have got here"); fail("Should not have got here");