big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10852. Optimise CSConfiguration getAllUserWeightsForQueue (#3392)

Browse Source
This commit is contained in:
9uapaw 2021-09-10 16:59:46 +02:00 committed by GitHub
parent b229e5a345
commit 811fd23f23
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 25 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
View File

@ -1126,7 +1126,7 @@ private static int[] findSubVariable(String eval) {
* @throws IllegalArgumentException when more than * @throws IllegalArgumentException when more than
* {@link Configuration#MAX_SUBST} replacements are required * {@link Configuration#MAX_SUBST} replacements are required
*/ */
private String substituteVars(String expr) { protected String substituteVars(String expr) {
if (expr == null) { if (expr == null) {
return null; return null;
} }

34
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
View File

@ -127,6 +127,13 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
@Private @Private
public static final String USER_SETTINGS = "user-settings"; public static final String USER_SETTINGS = "user-settings";
@Private
public static final String USER_WEIGHT_REGEX = "\\S+\\." + USER_WEIGHT;
@Private
public static final Pattern USER_WEIGHT_PATTERN = Pattern.compile(
USER_WEIGHT_REGEX);
@Private @Private
public static final float DEFAULT_USER_WEIGHT = 1.0f; public static final float DEFAULT_USER_WEIGHT = 1.0f;
@ -2031,18 +2038,23 @@ public void setPUOrderingPolicyUnderUtilizedPreemptionMoveReservation(
* @return map of user weights, if they exists. Otherwise, return empty map. * @return map of user weights, if they exists. Otherwise, return empty map.
*/ */
public Map<String, Float> getAllUserWeightsForQueue(String queuePath) { public Map<String, Float> getAllUserWeightsForQueue(String queuePath) {
Map <String, Float> userWeights = new HashMap <String, Float>(); Map <String, Float> userWeights = new HashMap <>();
String qPathPlusPrefix = String qPathPlusPrefix = getQueuePrefix(queuePath) + USER_SETTINGS;
getQueuePrefix(queuePath).replaceAll("\\.", "\\\\.") Map<String, String> props = getConfigurationProperties()
+ USER_SETTINGS + "\\."; .getPropertiesWithPrefix(qPathPlusPrefix);
String weightKeyRegex =
qPathPlusPrefix + "\\S+\\." + USER_WEIGHT; Map<String, String> result = new HashMap<>();
Map<String, String> props = getValByRegex(weightKeyRegex); for(Map.Entry<String, String> item: props.entrySet()) {
for (Entry<String, String> e : props.entrySet()) { Matcher m = USER_WEIGHT_PATTERN.matcher(item.getKey());
if(m.find()) {
result.put(item.getKey(), substituteVars(item.getValue()));
}
}
for (Entry<String, String> e : result.entrySet()) {
String userName = String userName =
e.getKey().replaceFirst(qPathPlusPrefix, "") e.getKey().replaceFirst("\\." + USER_WEIGHT, "");
.replaceFirst("\\." + USER_WEIGHT, ""); if (!userName.isEmpty()) {
if (userName != null && !userName.isEmpty()) {
userWeights.put(userName, new Float(e.getValue())); userWeights.put(userName, new Float(e.getValue()));
} }
} }

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
View File

@ -1853,6 +1853,7 @@ public void testUserSpecificUserLimits() throws Exception {
+ ".user-settings.firstname.lastname." + ".user-settings.firstname.lastname."
+ CapacitySchedulerConfiguration.USER_WEIGHT, + CapacitySchedulerConfiguration.USER_WEIGHT,
0.7f); 0.7f);
csConf.reinitializeConfigurationProperties();
when(csContext.getClusterResource()) when(csContext.getClusterResource())
.thenReturn(Resources.createResource(16 * GB, 32)); .thenReturn(Resources.createResource(16 * GB, 32));