big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-18021. Provide a public wrapper of Configuration#substituteVars (#3710)

Browse Source 
Contributed by Andras Gyori
This commit is contained in:
Andras Gyori 2021-12-03 17:44:58 +01:00 committed by GitHub
parent dd6b987c93
commit 47ea0d734f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 31 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
View File

@ -1099,6 +1099,20 @@ private static int[] findSubVariable(String eval) {
return result;
}
/**
* Provides a public wrapper over substituteVars in order to avoid compatibility issues.
* See HADOOP-18021 for further details.
*
* @param expr the literal value of a config key
* @return null if expr is null, otherwise the value resulting from expanding
* expr using the algorithm above.
* @throws IllegalArgumentException when more than
* {@link Configuration#MAX_SUBST} replacements are required
*/
public String substituteCommonVariables(String expr) {
return substituteVars(expr);
}
/**
* Attempts to repeatedly expand the value {@code expr} by replacing the
* left-most substring of the form "${var}" in the following precedence order
@ -1120,13 +1134,17 @@ private static int[] findSubVariable(String eval) {
* If a cycle is detected then the original expr is returned. Loops
* involving multiple substitutions are not detected.
*
* In order not to introduce breaking changes (as Oozie for example contains a method with the
* same name and same signature) do not make this method public, use substituteCommonVariables
* in this case.
*
* @param expr the literal value of a config key
* @return null if expr is null, otherwise the value resulting from expanding
* expr using the algorithm above.
* @throws IllegalArgumentException when more than
* {@link Configuration#MAX_SUBST} replacements are required
*/
public String substituteVars(String expr) {
private String substituteVars(String expr) {
if (expr == null) {
return null;
}

11
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
View File

@ -444,6 +444,17 @@ public void testVariableSubstitution() throws IOException {
assertTrue(mock.getInt("my.int", -1) == 42);
}
/**
* Checks if variable substitution is accessible via a public API.
*/
@Test
public void testCommonVariableSubstitution() {
conf.set("intvar", String.valueOf(42));
String intVar = conf.substituteCommonVariables("${intvar}");
assertEquals("42", intVar);
}
@Test
public void testEnvDefault() throws IOException {
Configuration mock = Mockito.spy(conf);

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserWeights.java
View File

@ -54,7 +54,7 @@ public static UserWeights createByConfig(
if (m.find()) {
String userName = item.getKey().replaceFirst("\\." + USER_WEIGHT, "");
if (!userName.isEmpty()) {
String value = conf.substituteVars(item.getValue());
String value = conf.substituteCommonVariables(item.getValue());
userWeights.data.put(userName, new Float(value));
}
}