big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10654. Dots '.' in CSMappingRule path variables should be replaced. Contributed by Peter Bacsko

Browse Source
This commit is contained in:
Szilard Nemeth 2021-04-23 16:07:58 +02:00
parent 0b04c9694d
commit f76a2a7606
2 changed files with 77 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/CSMappingPlacementRule.java
View File

@ -42,8 +42,6 @@
import java.util.List;
import java.util.Set;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT;
/**
* This class is responsible for making application submissions to queue
* assignments, based on the configured ruleset. This class supports all
@ -55,6 +53,8 @@
public class CSMappingPlacementRule extends PlacementRule {
private static final Logger LOG = LoggerFactory
.getLogger(CSMappingPlacementRule.class);
private static final String DOT = ".";
private static final String DOT_REPLACEMENT = "_dot_";
private CapacitySchedulerQueueManager queueManager;
private List<MappingRule> mappingRules;
@ -194,12 +194,13 @@ private void setupGroupsForVariableContext(VariableContext vctx, String user)
return;
}
Iterator<String> it = groupsSet.iterator();
String primaryGroup = it.next();
String primaryGroup = cleanName(it.next());
ArrayList<String> secondaryGroupList = new ArrayList<>();
while (it.hasNext()) {
secondaryGroupList.add(it.next());
String groupName = cleanName(it.next());
secondaryGroupList.add(groupName);
}
if (secondaryGroupList.size() == 0) {
@ -226,7 +227,7 @@ private VariableContext createVariableContext(
ApplicationSubmissionContext asc, String user) {
VariableContext vctx = new VariableContext();
vctx.put("%user", user);
vctx.put("%user", cleanName(user));
//If the specified matches the default it means NO queue have been specified
//as per ClientRMService#submitApplication which sets the queue to default
//when no queue is provided.
@ -522,4 +523,15 @@ private ApplicationPlacementContext placeToDefault(
" mapping rule. Please see the logs for details");
}
}
private String cleanName(String name) {
if (name.contains(DOT)) {
String converted = name.replaceAll("\\.", DOT_REPLACEMENT);
LOG.warn("Name {} is converted to {} when it is used as a queue name.",
name, converted);
return converted;
} else {
return name;
}
}
}

67
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/csmappingrule/TestCSMappingPlacementRule.java
View File

@ -63,13 +63,15 @@ public class TestCSMappingPlacementRule {
@Rule
public TemporaryFolder folder = new TemporaryFolder();
private Map<String, Set<String>> userGroups = ImmutableMap.of(
"alice", ImmutableSet.of("p_alice", "unique", "user"),
"bob", ImmutableSet.of("p_bob", "user", "developer"),
"charlie", ImmutableSet.of("p_charlie", "user", "tester"),
"dave", ImmutableSet.of("user"),
"emily", ImmutableSet.of("user", "tester", "developer")
);
private Map<String, Set<String>> userGroups =
ImmutableMap.<String, Set<String>>builder()
.put("alice", ImmutableSet.of("p_alice", "unique", "user"))
.put("bob", ImmutableSet.of("p_bob", "user", "developer"))
.put("charlie", ImmutableSet.of("p_charlie", "user", "tester"))
.put("dave", ImmutableSet.of("user"))
.put("emily", ImmutableSet.of("user", "tester", "developer"))
.put("test.user", ImmutableSet.of("main.grp", "sec.test.grp"))
.build();
private void createQueueHierarchy(CapacitySchedulerQueueManager queueManager) {
MockQueueHierarchyBuilder.create()
@ -79,6 +81,9 @@ private void createQueueHierarchy(CapacitySchedulerQueueManager queueManager) {
.withManagedParentQueue("root.man")
.withQueue("root.user.alice")
.withQueue("root.user.bob")
.withQueue("root.user.test_dot_user")
.withQueue("root.groups.main_dot_grp")
.withQueue("root.groups.sec_dot_test_dot_grp")
.withQueue("root.secondaryTests.unique")
.withQueue("root.secondaryTests.user")
.withQueue("root.ambiguous.user.charlie")
@ -718,4 +723,52 @@ public void testJSONFileConfiguration() throws IOException {
assertConfigTestResult(rules);
}
@Test
public void testUserNameCleanup() throws IOException {
ArrayList<MappingRule> rules = new ArrayList<>();
rules.add(
new MappingRule(
MappingRuleMatchers.createAllMatcher(),
(new MappingRuleActions.PlaceToQueueAction("%user", true))
.setFallbackReject()));
CSMappingPlacementRule engine = setupEngine(true, rules);
ApplicationSubmissionContext app = createApp("app");
assertPlace(
"test.user should be placed to root.users.test_dot_user",
engine, app, "test.user", "root.user.test_dot_user");
}
@Test
public void testPrimaryGroupNameCleanup() throws IOException {
ArrayList<MappingRule> rules = new ArrayList<>();
rules.add(
new MappingRule(
MappingRuleMatchers.createAllMatcher(),
(new MappingRuleActions.PlaceToQueueAction("%primary_group", true))
.setFallbackReject()));
CSMappingPlacementRule engine = setupEngine(true, rules);
ApplicationSubmissionContext app = createApp("app");
assertPlace(
"Application should have been placed to root.groups.main_dot_grp",
engine, app, "test.user", "root.groups.main_dot_grp");
}
@Test
public void testSecondaryGroupNameCleanup() throws IOException {
ArrayList<MappingRule> rules = new ArrayList<>();
rules.add(
new MappingRule(
MappingRuleMatchers.createAllMatcher(),
(new MappingRuleActions.PlaceToQueueAction("%secondary_group", true))
.setFallbackReject()));
CSMappingPlacementRule engine = setupEngine(true, rules);
ApplicationSubmissionContext app = createApp("app");
assertPlace(
"Application should have been placed to root.groups.sec_dot_test_dot_grp",
engine, app, "test.user", "root.groups.sec_dot_test_dot_grp");
}
}