big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-9840. Capacity scheduler: add support for Secondary Group rule mapping. Contributed by Manikandan R

Browse Source
This commit is contained in:
Szilard Nemeth 2019-10-15 17:22:52 +02:00
parent 336abbd873
commit a8ef03e961
5 changed files with 116 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/QueuePlacementRuleUtils.java
View File

@ -36,6 +36,8 @@ public final class QueuePlacementRuleUtils {
public static final String PRIMARY_GROUP_MAPPING = "%primary_group";
public static final String SECONDARY_GROUP_MAPPING = "%secondary_group";
private QueuePlacementRuleUtils() {
}
@ -100,7 +102,8 @@ public static QueueMappingEntity validateAndGetQueueMapping(
public static boolean isStaticQueueMapping(QueueMappingEntity mapping) {
return !mapping.getQueue().contains(CURRENT_USER_MAPPING) && !mapping
.getQueue().contains(PRIMARY_GROUP_MAPPING);
.getQueue().contains(PRIMARY_GROUP_MAPPING)
&& !mapping.getQueue().contains(SECONDARY_GROUP_MAPPING);
}
public static QueuePath extractQueuePath(String queueName)

43
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
View File

@ -54,9 +54,12 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
public static final String PRIMARY_GROUP_MAPPING = "%primary_group";
public static final String SECONDARY_GROUP_MAPPING = "%secondary_group";
private boolean overrideWithQueueMappings = false;
private List<QueueMapping> mappings = null;
private Groups groups;
private CapacitySchedulerQueueManager queueManager;
@Private
public static class QueueMapping {
@ -163,6 +166,27 @@ private ApplicationPlacementContext getPlacementForUser(String user)
return getPlacementContext(mapping, user);
} else if (mapping.queue.equals(PRIMARY_GROUP_MAPPING)) {
return getPlacementContext(mapping, groups.getGroups(user).get(0));
} else if (mapping.queue.equals(SECONDARY_GROUP_MAPPING)) {
List<String> groupsList = groups.getGroups(user);
String secondaryGroup = null;
// Traverse all secondary groups (as there could be more than one
// and position is not guaranteed) and ensure there is queue with
// the same name
for (int i = 1; i < groupsList.size(); i++) {
if (this.queueManager.getQueue(groupsList.get(i)) != null) {
secondaryGroup = groupsList.get(i);
break;
}
}
if (secondaryGroup != null) {
return getPlacementContext(mapping, secondaryGroup);
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("User {} is not associated with any Secondary "
+ "Group. Hence it may use the 'default' queue", user);
}
return null;
}
} else {
return getPlacementContext(mapping);
}
@ -251,8 +275,7 @@ public boolean initialize(ResourceScheduler scheduler)
// Get new user/group mappings
List<QueueMapping> newMappings = new ArrayList<>();
CapacitySchedulerQueueManager queueManager =
schedulerContext.getCapacitySchedulerQueueManager();
queueManager = schedulerContext.getCapacitySchedulerQueueManager();
// check if mappings refer to valid queues
for (QueueMapping mapping : queueMappings) {
@ -365,10 +388,12 @@ private static QueueMapping validateAndGetAutoCreatedQueueMapping(
}
private static boolean isStaticQueueMapping(QueueMapping mapping) {
return !mapping.getQueue().contains(
UserGroupMappingPlacementRule.CURRENT_USER_MAPPING) && !mapping
.getQueue().contains(
UserGroupMappingPlacementRule.PRIMARY_GROUP_MAPPING);
return !mapping.getQueue()
.contains(UserGroupMappingPlacementRule.CURRENT_USER_MAPPING)
&& !mapping.getQueue()
.contains(UserGroupMappingPlacementRule.PRIMARY_GROUP_MAPPING)
&& !mapping.getQueue()
.contains(UserGroupMappingPlacementRule.SECONDARY_GROUP_MAPPING);
}
private static class QueuePath {
@ -443,4 +468,10 @@ private static void validateParentQueue(CSQueue parentQueue,
public List<QueueMapping> getQueueMappings() {
return mappings;
}
@VisibleForTesting
@Private
public void setQueueManager(CapacitySchedulerQueueManager queueManager) {
this.queueManager = queueManager;
}
}

27
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java
View File

@ -18,6 +18,9 @@
package org.apache.hadoop.yarn.server.resourcemanager.placement;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.util.Arrays;
import org.apache.hadoop.fs.CommonConfigurationKeys;
@ -28,6 +31,9 @@
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping;
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping.MappingType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.PrimaryGroupMapping;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.SimpleGroupsMapping;
import org.apache.hadoop.yarn.util.Records;
import org.junit.Assert;
@ -54,6 +60,10 @@ private void verifyQueueMapping(QueueMapping queueMapping, String inputUser,
Groups groups = new Groups(conf);
UserGroupMappingPlacementRule rule = new UserGroupMappingPlacementRule(
overwrite, Arrays.asList(queueMapping), groups);
CapacitySchedulerQueueManager queueManager =
mock(CapacitySchedulerQueueManager.class);
when(queueManager.getQueue("asubgroup2")).thenReturn(mock(CSQueue.class));
rule.setQueueManager(queueManager);
ApplicationSubmissionContext asc = Records.newRecord(
ApplicationSubmissionContext.class);
asc.setQueue(inputQueue);
@ -62,9 +72,24 @@ private void verifyQueueMapping(QueueMapping queueMapping, String inputUser,
ctx != null ? ctx.getQueue() : inputQueue);
}
@Test
public void testSecondaryGroupMapping() throws YarnException {
verifyQueueMapping(
new QueueMapping(MappingType.USER, "%user", "%secondary_group"), "a",
"asubgroup2");
// PrimaryGroupMapping.class returns only primary group, no secondary groups
conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
PrimaryGroupMapping.class, GroupMappingServiceProvider.class);
verifyQueueMapping(
new QueueMapping(MappingType.USER, "%user", "%secondary_group"), "a",
"default");
}
@Test
public void testMapping() throws YarnException {
// simple base case for mapping user to queue
verifyQueueMapping(new QueueMapping(MappingType.USER, "a", "q1"), "a", "q1");
verifyQueueMapping(new QueueMapping(MappingType.GROUP, "agroup", "q1"),
"a", "q1");

47
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/PrimaryGroupMapping.java Normal file
View File

@ -0,0 +1,47 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import org.apache.hadoop.security.GroupMappingServiceProvider;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
/**
* Group Mapping class used for test cases. Returns only primary group of the
* given user
*/
public class PrimaryGroupMapping implements GroupMappingServiceProvider {
@Override
public List<String> getGroups(String user) {
return Arrays.asList(user + "group");
}
@Override
public void cacheGroupsRefresh() throws IOException {
throw new UnsupportedOperationException();
}
@Override
public void cacheGroupsAdd(List<String> groups) throws IOException {
throw new UnsupportedOperationException();
}
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
View File

@ -161,7 +161,7 @@ Configuration
| Property | Description |
|:---- |:---- |
| `yarn.scheduler.capacity.queue-mappings` | This configuration specifies the mapping of user or group to a specific queue. You can map a single user or a list of users to queues. Syntax: `[u or g]:[name]:[queue_name][,next_mapping]*`. Here, *u or g* indicates whether the mapping is for a user or group. The value is *u* for user and *g* for group. *name* indicates the user name or group name. To specify the user who has submitted the application, %user can be used. *queue_name* indicates the queue name for which the application has to be mapped. To specify queue name same as user name, *%user* can be used. To specify queue name same as the name of the primary group for which the user belongs to, *%primary_group* can be used.|
| `yarn.scheduler.capacity.queue-mappings` | This configuration specifies the mapping of user or group to a specific queue. You can map a single user or a list of users to queues. Syntax: `[u or g]:[name]:[queue_name][,next_mapping]*`. Here, *u or g* indicates whether the mapping is for a user or group. The value is *u* for user and *g* for group. *name* indicates the user name or group name. To specify the user who has submitted the application, %user can be used. *queue_name* indicates the queue name for which the application has to be mapped. To specify queue name same as user name, *%user* can be used. To specify queue name same as the name of the primary group for which the user belongs to, *%primary_group* can be used. Secondary group can be referenced as *%secondary_group* |
| `yarn.scheduler.queue-placement-rules.app-name` | This configuration specifies the mapping of application_name to a specific queue. You can map a single application or a list of applications to queues. Syntax: `[app_name]:[queue_name][,next_mapping]*`. Here, *app_name* indicates the application name you want to do the mapping. *queue_name* indicates the queue name for which the application has to be mapped. To specify the current application's name as the app_name, %application can be used.|
| `yarn.scheduler.capacity.queue-mappings-override.enable` | This function is used to specify whether the user specified queues can be overridden. This is a Boolean value and the default value is *false*. |
@ -170,7 +170,7 @@ Example:
```
<property>
<name>yarn.scheduler.capacity.queue-mappings</name>
<value>u:user1:queue1,g:group1:queue2,u:%user:%user,u:user2:%primary_group</value>
<value>u:user1:queue1,g:group1:queue2,u:%user:%user,u:user2:%primary_group,u:user3:%secondary_group</value>
<description>
Here, <user1> is mapped to <queue1>, <group1> is mapped to <queue2>,
maps users to queues with the same name as user, <user2> is mapped