big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10652. Capacity Scheduler fails to handle user weights for a user that has a "." (dot) in it

Browse Source 
This only fixes the user name resolution for weights in the queues. It
does not add generic support for user names with dots in all use cases
in the capacity scheduler.

Contributed by: Siddharth Ahuja
This commit is contained in:
Wilfred Spiegelenburg 2021-03-17 10:47:49 +11:00
parent 47620f8821
commit f276f1af80
No known key found for this signature in database
GPG Key ID: 7BB711D7AC5E7F29
2 changed files with 11 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
View File

@ -1963,7 +1963,7 @@ public Map<String, Float> getAllUserWeightsForQueue(String queuePath) {
getQueuePrefix(queuePath).replaceAll("\\.", "\\\\.") getQueuePrefix(queuePath).replaceAll("\\.", "\\\\.")
+ USER_SETTINGS + "\\."; + USER_SETTINGS + "\\.";
String weightKeyRegex = String weightKeyRegex =
qPathPlusPrefix + "\\w+\\." + USER_WEIGHT; qPathPlusPrefix + "\\S+\\." + USER_WEIGHT;
Map<String, String> props = getValByRegex(weightKeyRegex); Map<String, String> props = getValByRegex(weightKeyRegex);
for (Entry<String, String> e : props.entrySet()) { for (Entry<String, String> e : props.entrySet()) {
String userName = String userName =

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
View File

@ -1660,17 +1660,25 @@ public void testUserSpecificUserLimits() throws Exception {
LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A)); LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
// Set minimum-user-limit-percent for queue "a" in the configs. // Set minimum-user-limit-percent for queue "a" in the configs.
csConf.setUserLimit(a.getQueuePath(), 50); csConf.setUserLimit(a.getQueuePath(), 50);
// Set weight for "user_0" to be 1.5 for the a queue in the configs. // Set weight for "user_0" to be 1.5f for the a queue in the configs.
csConf.setFloat("yarn.scheduler.capacity." + a.getQueuePath() csConf.setFloat("yarn.scheduler.capacity." + a.getQueuePath()
+ ".user-settings.user_0." + CapacitySchedulerConfiguration.USER_WEIGHT, + ".user-settings.user_0." + CapacitySchedulerConfiguration.USER_WEIGHT,
1.5f); 1.5f);
// Set weight for "firstname.lastname" to be 0.7f for the a queue
// in the configs. Notice the user contains a dot. This is to test
// that weights are accepted for a username that contains dots.
csConf.setFloat("yarn.scheduler.capacity." + a.getQueuePath()
+ ".user-settings.firstname.lastname."
+ CapacitySchedulerConfiguration.USER_WEIGHT,
0.7f);
when(csContext.getClusterResource()) when(csContext.getClusterResource())
.thenReturn(Resources.createResource(16 * GB, 32)); .thenReturn(Resources.createResource(16 * GB, 32));
// Verify that configs were updated and parsed correctly. // Verify that configs were updated and parsed correctly.
Assert.assertNull(a.getUserWeights().get("user_0")); Assert.assertNull(a.getUserWeights().get("user_0"));
a.reinitialize(a, csContext.getClusterResource()); a.reinitialize(a, csContext.getClusterResource());
assertEquals(1.5, a.getUserWeights().get("user_0").floatValue(), 0.0); assertEquals(1.5f, a.getUserWeights().get("user_0"), 0.0f);
assertEquals(0.7f, a.getUserWeights().get("firstname.lastname"), 0.0f);
// set maxCapacity // set maxCapacity
a.setMaxCapacity(1.0f); a.setMaxCapacity(1.0f);