diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java index 894bc82622..9648769231 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java @@ -42,8 +42,6 @@ import java.util.List; import java.util.Set; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT; - /** * This class is responsible for making application submissions to queue * assignments, based on the configured ruleset. This class supports all @@ -55,6 +53,8 @@ public class CSMappingPlacementRule extends PlacementRule { private static final Logger LOG = LoggerFactory .getLogger(CSMappingPlacementRule.class); + private static final String DOT = "."; + private static final String DOT_REPLACEMENT = "_dot_"; private CapacitySchedulerQueueManager queueManager; private List mappingRules; @@ -194,12 +194,13 @@ private void setupGroupsForVariableContext(VariableContext vctx, String user) return; } Iterator it = groupsSet.iterator(); - String primaryGroup = it.next(); + String primaryGroup = cleanName(it.next()); ArrayList secondaryGroupList = new ArrayList<>(); while (it.hasNext()) { - secondaryGroupList.add(it.next()); + String groupName = cleanName(it.next()); + secondaryGroupList.add(groupName); } if (secondaryGroupList.size() == 0) { @@ -226,7 +227,7 @@ private VariableContext createVariableContext( ApplicationSubmissionContext asc, String user) { VariableContext vctx = new VariableContext(); - vctx.put("%user", user); + vctx.put("%user", cleanName(user)); //If the specified matches the default it means NO queue have been specified //as per ClientRMService#submitApplication which sets the queue to default //when no queue is provided. @@ -522,4 +523,15 @@ private ApplicationPlacementContext placeToDefault( " mapping rule. Please see the logs for details"); } } + + private String cleanName(String name) { + if (name.contains(DOT)) { + String converted = name.replaceAll("\\.", DOT_REPLACEMENT); + LOG.warn("Name {} is converted to {} when it is used as a queue name.", + name, converted); + return converted; + } else { + return name; + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/TestCSMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/TestCSMappingPlacementRule.java index f6bb2a1587..0cf10595ee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/TestCSMappingPlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/TestCSMappingPlacementRule.java @@ -63,13 +63,15 @@ public class TestCSMappingPlacementRule { @Rule public TemporaryFolder folder = new TemporaryFolder(); - private Map> userGroups = ImmutableMap.of( - "alice", ImmutableSet.of("p_alice", "unique", "user"), - "bob", ImmutableSet.of("p_bob", "user", "developer"), - "charlie", ImmutableSet.of("p_charlie", "user", "tester"), - "dave", ImmutableSet.of("user"), - "emily", ImmutableSet.of("user", "tester", "developer") - ); + private Map> userGroups = + ImmutableMap.>builder() + .put("alice", ImmutableSet.of("p_alice", "unique", "user")) + .put("bob", ImmutableSet.of("p_bob", "user", "developer")) + .put("charlie", ImmutableSet.of("p_charlie", "user", "tester")) + .put("dave", ImmutableSet.of("user")) + .put("emily", ImmutableSet.of("user", "tester", "developer")) + .put("test.user", ImmutableSet.of("main.grp", "sec.test.grp")) + .build(); private void createQueueHierarchy(CapacitySchedulerQueueManager queueManager) { MockQueueHierarchyBuilder.create() @@ -79,6 +81,9 @@ private void createQueueHierarchy(CapacitySchedulerQueueManager queueManager) { .withManagedParentQueue("root.man") .withQueue("root.user.alice") .withQueue("root.user.bob") + .withQueue("root.user.test_dot_user") + .withQueue("root.groups.main_dot_grp") + .withQueue("root.groups.sec_dot_test_dot_grp") .withQueue("root.secondaryTests.unique") .withQueue("root.secondaryTests.user") .withQueue("root.ambiguous.user.charlie") @@ -718,4 +723,52 @@ public void testJSONFileConfiguration() throws IOException { assertConfigTestResult(rules); } + + @Test + public void testUserNameCleanup() throws IOException { + ArrayList rules = new ArrayList<>(); + rules.add( + new MappingRule( + MappingRuleMatchers.createAllMatcher(), + (new MappingRuleActions.PlaceToQueueAction("%user", true)) + .setFallbackReject())); + + CSMappingPlacementRule engine = setupEngine(true, rules); + ApplicationSubmissionContext app = createApp("app"); + assertPlace( + "test.user should be placed to root.users.test_dot_user", + engine, app, "test.user", "root.user.test_dot_user"); + } + + @Test + public void testPrimaryGroupNameCleanup() throws IOException { + ArrayList rules = new ArrayList<>(); + rules.add( + new MappingRule( + MappingRuleMatchers.createAllMatcher(), + (new MappingRuleActions.PlaceToQueueAction("%primary_group", true)) + .setFallbackReject())); + + CSMappingPlacementRule engine = setupEngine(true, rules); + ApplicationSubmissionContext app = createApp("app"); + assertPlace( + "Application should have been placed to root.groups.main_dot_grp", + engine, app, "test.user", "root.groups.main_dot_grp"); + } + + @Test + public void testSecondaryGroupNameCleanup() throws IOException { + ArrayList rules = new ArrayList<>(); + rules.add( + new MappingRule( + MappingRuleMatchers.createAllMatcher(), + (new MappingRuleActions.PlaceToQueueAction("%secondary_group", true)) + .setFallbackReject())); + + CSMappingPlacementRule engine = setupEngine(true, rules); + ApplicationSubmissionContext app = createApp("app"); + assertPlace( + "Application should have been placed to root.groups.sec_dot_test_dot_grp", + engine, app, "test.user", "root.groups.sec_dot_test_dot_grp"); + } } \ No newline at end of file