big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixing Job History Server Configuration parsing. (Jason Lowe via asuresh)

Browse Source
This commit is contained in:
Arun Suresh 2017-11-09 15:15:51 -08:00
parent dd07038ffa
commit 5eb7dbe9b3
2 changed files with 122 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

163
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.conf;
import com.ctc.wstx.api.ReaderConfig;
import com.ctc.wstx.io.StreamBootstrapper;
import com.ctc.wstx.io.SystemId;
import com.ctc.wstx.stax.WstxInputFactory;
@ -70,6 +71,7 @@
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamConstants;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
@ -91,6 +93,7 @@
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProvider.CredentialEntry;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
@ -206,19 +209,31 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
private static final String DEFAULT_STRING_CHECK =
"testingforemptydefaultvalue";
private static boolean restrictSystemPropsDefault = false;
private boolean restrictSystemProps = restrictSystemPropsDefault;
private boolean allowNullValueProperties = false;
private static class Resource {
private final Object resource;
private final String name;
private final boolean restrictParser;
public Resource(Object resource) {
this(resource, resource.toString());
}
public Resource(Object resource, boolean useRestrictedParser) {
this(resource, resource.toString(), useRestrictedParser);
}
public Resource(Object resource, String name) {
this(resource, name, getRestrictParserDefault(resource));
}
public Resource(Object resource, String name, boolean restrictParser) {
this.resource = resource;
this.name = name;
this.restrictParser = restrictParser;
}
public String getName(){
@ -228,11 +243,28 @@ public String getName(){
public Object getResource() {
return resource;
}
public boolean isParserRestricted() {
return restrictParser;
}
@Override
public String toString() {
return name;
}
private static boolean getRestrictParserDefault(Object resource) {
if (resource instanceof String) {
return false;
}
UserGroupInformation user;
try {
user = UserGroupInformation.getCurrentUser();
} catch (IOException e) {
throw new RuntimeException("Unable to determine current user", e);
}
return user.getRealUser() != null;
}
}
/**
@ -254,7 +286,7 @@ public String toString() {
new ConcurrentHashMap<String, Boolean>());
private boolean loadDefaults = true;
/**
* Configuration objects
*/
@ -777,6 +809,7 @@ public Configuration(Configuration other) {
this.overlay = (Properties)other.overlay.clone();
}
this.restrictSystemProps = other.restrictSystemProps;
if (other.updatingResource != null) {
this.updatingResource = new ConcurrentHashMap<String, String[]>(
other.updatingResource);
@ -825,6 +858,14 @@ public static synchronized void addDefaultResource(String name) {
}
}
public static void setRestrictSystemPropertiesDefault(boolean val) {
restrictSystemPropsDefault = val;
}
public void setRestrictSystemProperties(boolean val) {
this.restrictSystemProps = val;
}
/**
* Add a configuration resource.
*
@ -838,6 +879,10 @@ public void addResource(String name) {
addResourceObject(new Resource(name));
}
public void addResource(String name, boolean restrictedParser) {
addResourceObject(new Resource(name, restrictedParser));
}
/**
* Add a configuration resource.
*
@ -852,6 +897,10 @@ public void addResource(URL url) {
addResourceObject(new Resource(url));
}
public void addResource(URL url, boolean restrictedParser) {
addResourceObject(new Resource(url, restrictedParser));
}
/**
* Add a configuration resource.
*
@ -866,6 +915,10 @@ public void addResource(Path file) {
addResourceObject(new Resource(file));
}
public void addResource(Path file, boolean restrictedParser) {
addResourceObject(new Resource(file, restrictedParser));
}
/**
* Add a configuration resource.
*
@ -883,6 +936,10 @@ public void addResource(InputStream in) {
addResourceObject(new Resource(in));
}
public void addResource(InputStream in, boolean restrictedParser) {
addResourceObject(new Resource(in, restrictedParser));
}
/**
* Add a configuration resource.
*
@ -896,7 +953,12 @@ public void addResource(InputStream in) {
public void addResource(InputStream in, String name) {
addResourceObject(new Resource(in, name));
}
public void addResource(InputStream in, String name,
boolean restrictedParser) {
addResourceObject(new Resource(in, name, restrictedParser));
}
/**
* Add a configuration resource.
*
@ -906,7 +968,7 @@ public void addResource(InputStream in, String name) {
* @param conf Configuration object from which to load properties
*/
public void addResource(Configuration conf) {
addResourceObject(new Resource(conf.getProps()));
addResourceObject(new Resource(conf.getProps(), conf.restrictSystemProps));
}
@ -926,6 +988,7 @@ public synchronized void reloadConfiguration() {
private synchronized void addResourceObject(Resource resource) {
resources.add(resource); // add to resources
restrictSystemProps |= resource.isParserRestricted();
reloadConfiguration();
}
@ -1034,34 +1097,36 @@ private String substituteVars(String expr) {
final String var = eval.substring(varBounds[SUB_START_IDX],
varBounds[SUB_END_IDX]);
String val = null;
try {
if (var.startsWith("env.") && 4 < var.length()) {
String v = var.substring(4);
int i = 0;
for (; i < v.length(); i++) {
char c = v.charAt(i);
if (c == ':' && i < v.length() - 1 && v.charAt(i + 1) == '-') {
val = getenv(v.substring(0, i));
if (val == null || val.length() == 0) {
val = v.substring(i + 2);
if (!restrictSystemProps) {
try {
if (var.startsWith("env.") && 4 < var.length()) {
String v = var.substring(4);
int i = 0;
for (; i < v.length(); i++) {
char c = v.charAt(i);
if (c == ':' && i < v.length() - 1 && v.charAt(i + 1) == '-') {
val = getenv(v.substring(0, i));
if (val == null || val.length() == 0) {
val = v.substring(i + 2);
}
break;
} else if (c == '-') {
val = getenv(v.substring(0, i));
if (val == null) {
val = v.substring(i + 1);
}
break;
}
break;
} else if (c == '-') {
val = getenv(v.substring(0, i));
if (val == null) {
val = v.substring(i + 1);
}
break;
}
if (i == v.length()) {
val = getenv(v);
}
} else {
val = getProperty(var);
}
if (i == v.length()) {
val = getenv(v);
}
} else {
val = getProperty(var);
} catch (SecurityException se) {
LOG.warn("Unexpected SecurityException in Configuration", se);
}
} catch(SecurityException se) {
LOG.warn("Unexpected SecurityException in Configuration", se);
}
if (val == null) {
val = getRaw(var);
@ -1128,6 +1193,10 @@ public void setAllowNullValueProperties( boolean val ) {
this.allowNullValueProperties = val;
}
public void setRestrictSystemProps(boolean val) {
this.restrictSystemProps = val;
}
/**
* Return existence of the <code>name</code> property, but only for
* names which have no valid value, usually non-existent or commented
@ -2718,7 +2787,7 @@ public Map<String, String> getPropsWithPrefix(String confPrefix) {
return configMap;
}
private XMLStreamReader parse(URL url)
private XMLStreamReader parse(URL url, boolean restricted)
throws IOException, XMLStreamException {
if (!quietmode) {
if (LOG.isDebugEnabled()) {
@ -2735,11 +2804,11 @@ private XMLStreamReader parse(URL url)
// with other users.
connection.setUseCaches(false);
}
return parse(connection.getInputStream(), url.toString());
return parse(connection.getInputStream(), url.toString(), restricted);
}
private XMLStreamReader parse(InputStream is, String systemIdStr)
throws IOException, XMLStreamException {
private XMLStreamReader parse(InputStream is, String systemIdStr,
boolean restricted) throws IOException, XMLStreamException {
if (!quietmode) {
LOG.debug("parsing input stream " + is);
}
@ -2747,9 +2816,12 @@ private XMLStreamReader parse(InputStream is, String systemIdStr)
return null;
}
SystemId systemId = SystemId.construct(systemIdStr);
return XML_INPUT_FACTORY.createSR(XML_INPUT_FACTORY.createPrivateConfig(),
systemId, StreamBootstrapper.getInstance(null, systemId, is), false,
true);
ReaderConfig readerConfig = XML_INPUT_FACTORY.createPrivateConfig();
if (restricted) {
readerConfig.setProperty(XMLInputFactory.SUPPORT_DTD, false);
}
return XML_INPUT_FACTORY.createSR(readerConfig, systemId,
StreamBootstrapper.getInstance(null, systemId, is), false, true);
}
private void loadResources(Properties properties,
@ -2757,7 +2829,7 @@ private void loadResources(Properties properties,
boolean quiet) {
if(loadDefaults) {
for (String resource : defaultResources) {
loadResource(properties, new Resource(resource), quiet);
loadResource(properties, new Resource(resource, false), quiet);
}
}
@ -2777,12 +2849,13 @@ private Resource loadResource(Properties properties,
name = wrapper.getName();
XMLStreamReader2 reader = null;
boolean returnCachedProperties = false;
boolean isRestricted = wrapper.isParserRestricted();
if (resource instanceof URL) { // an URL resource
reader = (XMLStreamReader2)parse((URL)resource);
reader = (XMLStreamReader2)parse((URL)resource, isRestricted);
} else if (resource instanceof String) { // a CLASSPATH resource
URL url = getResource((String)resource);
reader = (XMLStreamReader2)parse(url);
reader = (XMLStreamReader2)parse(url, isRestricted);
} else if (resource instanceof Path) { // a file resource
// Can't use FileSystem API or we get an infinite loop
// since FileSystem uses Configuration API. Use java.io.File instead.
@ -2793,10 +2866,12 @@ private Resource loadResource(Properties properties,
LOG.debug("parsing File " + file);
}
reader = (XMLStreamReader2)parse(new BufferedInputStream(
new FileInputStream(file)), ((Path)resource).toString());
new FileInputStream(file)), ((Path)resource).toString(),
isRestricted);
}
} else if (resource instanceof InputStream) {
reader = (XMLStreamReader2)parse((InputStream)resource, null);
reader = (XMLStreamReader2)parse((InputStream)resource, null,
isRestricted);
returnCachedProperties = true;
} else if (resource instanceof Properties) {
overlay(properties, (Properties)resource);
@ -2878,6 +2953,10 @@ private Resource loadResource(Properties properties,
if (confInclude == null) {
break;
}
if (isRestricted) {
throw new RuntimeException("Error parsing resource " + wrapper
+ ": XInclude is not supported for restricted resources");
}
// Determine if the included resource is a classpath resource
// otherwise fallback to a file resource
// xi:include are treated as inline and retain current source
@ -2992,7 +3071,7 @@ private Resource loadResource(Properties properties,
if (returnCachedProperties) {
overlay(properties, toAddTo);
return new Resource(toAddTo, name);
return new Resource(toAddTo, name, wrapper.isParserRestricted());
}
return null;
} catch (IOException e) {

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
View File

@ -508,7 +508,7 @@ public synchronized Path getConfFile() {
public synchronized Configuration loadConfFile() throws IOException {
FileContext fc = FileContext.getFileContext(confFile.toUri(), conf);
Configuration jobConf = new Configuration(false);
jobConf.addResource(fc.open(confFile), confFile.toString());
jobConf.addResource(fc.open(confFile), confFile.toString(), true);
return jobConf;
}