big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10852. Optimise CSConfiguration getAllUserWeightsForQueue (#3392)

Browse Source
This commit is contained in:
9uapaw 2021-09-10 16:59:46 +02:00 committed by GitHub
parent b229e5a345
commit 811fd23f23
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 25 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
View File

@ -1126,7 +1126,7 @@ private static int[] findSubVariable(String eval) {
* @throws IllegalArgumentException when more than
* {@link Configuration#MAX_SUBST} replacements are required
*/
private String substituteVars(String expr) {
protected String substituteVars(String expr) {
if (expr == null) {
return null;
}

34
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
View File

@ -127,6 +127,13 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
@Private
public static final String USER_SETTINGS = "user-settings";
@Private
public static final String USER_WEIGHT_REGEX = "\\S+\\." + USER_WEIGHT;
@Private
public static final Pattern USER_WEIGHT_PATTERN = Pattern.compile(
USER_WEIGHT_REGEX);
@Private
public static final float DEFAULT_USER_WEIGHT = 1.0f;
@ -2031,18 +2038,23 @@ public void setPUOrderingPolicyUnderUtilizedPreemptionMoveReservation(
* @return map of user weights, if they exists. Otherwise, return empty map.
*/
public Map<String, Float> getAllUserWeightsForQueue(String queuePath) {
Map <String, Float> userWeights = new HashMap <String, Float>();
String qPathPlusPrefix =
getQueuePrefix(queuePath).replaceAll("\\.", "\\\\.")
+ USER_SETTINGS + "\\.";
String weightKeyRegex =
qPathPlusPrefix + "\\S+\\." + USER_WEIGHT;
Map<String, String> props = getValByRegex(weightKeyRegex);
for (Entry<String, String> e : props.entrySet()) {
Map <String, Float> userWeights = new HashMap <>();
String qPathPlusPrefix = getQueuePrefix(queuePath) + USER_SETTINGS;
Map<String, String> props = getConfigurationProperties()
.getPropertiesWithPrefix(qPathPlusPrefix);
Map<String, String> result = new HashMap<>();
for(Map.Entry<String, String> item: props.entrySet()) {
Matcher m = USER_WEIGHT_PATTERN.matcher(item.getKey());
if(m.find()) {
result.put(item.getKey(), substituteVars(item.getValue()));
}
}
for (Entry<String, String> e : result.entrySet()) {
String userName =
e.getKey().replaceFirst(qPathPlusPrefix, "")
.replaceFirst("\\." + USER_WEIGHT, "");
if (userName != null && !userName.isEmpty()) {
e.getKey().replaceFirst("\\." + USER_WEIGHT, "");
if (!userName.isEmpty()) {
userWeights.put(userName, new Float(e.getValue()));
}
}

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
View File

@ -1853,6 +1853,7 @@ public void testUserSpecificUserLimits() throws Exception {
+ ".user-settings.firstname.lastname."
+ CapacitySchedulerConfiguration.USER_WEIGHT,
0.7f);
csConf.reinitializeConfigurationProperties();
when(csContext.getClusterResource())
.thenReturn(Resources.createResource(16 * GB, 32));