YARN-10585. Create a class which can convert from legacy mapping rule format to the new JSON format. Contributed by Gergely Pollak
This commit is contained in:
parent
4f008153ef
commit
e2a7008d50
@ -106,7 +106,11 @@ public static MappingRule createLegacyRule(
|
|||||||
|
|
||||||
switch (type) {
|
switch (type) {
|
||||||
case USER_MAPPING:
|
case USER_MAPPING:
|
||||||
|
if (source.equals("%user")) {
|
||||||
|
matcher = MappingRuleMatchers.createAllMatcher();
|
||||||
|
} else {
|
||||||
matcher = MappingRuleMatchers.createUserMatcher(source);
|
matcher = MappingRuleMatchers.createUserMatcher(source);
|
||||||
|
}
|
||||||
break;
|
break;
|
||||||
case GROUP_MAPPING:
|
case GROUP_MAPPING:
|
||||||
matcher = MappingRuleMatchers.createUserGroupMatcher(source);
|
matcher = MappingRuleMatchers.createUserGroupMatcher(source);
|
||||||
|
@ -96,8 +96,9 @@ public void validate(MappingRuleValidationContext ctx)
|
|||||||
@Override
|
@Override
|
||||||
public String toString() {
|
public String toString() {
|
||||||
return "PlaceToQueueAction{" +
|
return "PlaceToQueueAction{" +
|
||||||
"queueName='" + queuePattern + '\'' +
|
"queueName='" + queuePattern + "'," +
|
||||||
'}';
|
"allowCreate=" + allowCreate +
|
||||||
|
"}";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -0,0 +1,405 @@
|
|||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.converter;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.fasterxml.jackson.databind.node.ArrayNode;
|
||||||
|
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||||
|
import org.apache.hadoop.util.StringUtils;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingQueuePath;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.HashSet;
|
||||||
|
|
||||||
|
public class LegacyMappingRuleToJson {
|
||||||
|
//Legacy rule parse helper constants
|
||||||
|
public static final String RULE_PART_DELIMITER = ":";
|
||||||
|
public static final String PREFIX_USER_MAPPING = "u";
|
||||||
|
public static final String PREFIX_GROUP_MAPPING = "g";
|
||||||
|
|
||||||
|
//Legacy rule matcher variables
|
||||||
|
public static final String MATCHER_APPLICATION = "%application";
|
||||||
|
public static final String MATCHER_USER = "%user";
|
||||||
|
|
||||||
|
//Legacy rule mapping variables, which can be used in target queues
|
||||||
|
public static final String MAPPING_PRIMARY_GROUP = "%primary_group";
|
||||||
|
public static final String MAPPING_SECONDARY_GROUP = "%secondary_group";
|
||||||
|
public static final String MAPPING_USER = MATCHER_USER;
|
||||||
|
|
||||||
|
//JSON Format match all token (actually only used for users)
|
||||||
|
public static final String JSON_MATCH_ALL = "*";
|
||||||
|
|
||||||
|
//Frequently used JSON node names for rule definitions
|
||||||
|
public static final String JSON_NODE_POLICY = "policy";
|
||||||
|
public static final String JSON_NODE_PARENT_QUEUE = "parentQueue";
|
||||||
|
public static final String JSON_NODE_CUSTOM_PLACEMENT = "customPlacement";
|
||||||
|
public static final String JSON_NODE_MATCHES = "matches";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Our internal object mapper, used to create JSON nodes.
|
||||||
|
*/
|
||||||
|
private ObjectMapper objectMapper = new ObjectMapper();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Collection to store the legacy group mapping rule strings.
|
||||||
|
*/
|
||||||
|
private Collection<String> userGroupMappingRules = new ArrayList<>();
|
||||||
|
/**
|
||||||
|
* Collection to store the legacy application name mapping rule strings.
|
||||||
|
*/
|
||||||
|
private Collection<String> applicationNameMappingRules = new ArrayList<>();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This setter method is used to set the raw string format of the legacy
|
||||||
|
* user group mapping rules. This method expect a string formatted just like
|
||||||
|
* in the configuration file of the Capacity Scheduler.
|
||||||
|
* eg. u:bob:root.groups.%primary_group,u:%user:root.default
|
||||||
|
*
|
||||||
|
* @param rules The string containing ALL the UserGroup mapping rules in
|
||||||
|
* legacy format
|
||||||
|
* @return This object for daisy chain support
|
||||||
|
*/
|
||||||
|
public LegacyMappingRuleToJson setUserGroupMappingRules(String rules) {
|
||||||
|
setUserGroupMappingRules(StringUtils.getTrimmedStringCollection(rules));
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This setter method is used to set the the user group mapping rules as a
|
||||||
|
* string collection, where each entry is one rule.
|
||||||
|
*
|
||||||
|
* @param rules One rule per entry
|
||||||
|
* @return This object for daisy chain support
|
||||||
|
*/
|
||||||
|
public LegacyMappingRuleToJson setUserGroupMappingRules(
|
||||||
|
Collection<String> rules) {
|
||||||
|
if (rules != null) {
|
||||||
|
userGroupMappingRules = rules;
|
||||||
|
} else {
|
||||||
|
userGroupMappingRules = new ArrayList<>();
|
||||||
|
}
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This setter method is used to set the raw string format of the legacy
|
||||||
|
* application name mapping rules. This method expect a string formatted
|
||||||
|
* just like in the configuration file of the Capacity Scheduler.
|
||||||
|
* eg. mapreduce:root.apps.%application,%application:root.default
|
||||||
|
*
|
||||||
|
* @param rules The string containing ALL the application name mapping rules
|
||||||
|
* in legacy format
|
||||||
|
* @return This object for daisy chain support
|
||||||
|
*/
|
||||||
|
public LegacyMappingRuleToJson setAppNameMappingRules(String rules) {
|
||||||
|
setAppNameMappingRules(StringUtils.getTrimmedStringCollection(rules));
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This setter method is used to set the the application name mapping rules as
|
||||||
|
* a string collection, where each entry is one rule.
|
||||||
|
*
|
||||||
|
* @param rules One rule per entry
|
||||||
|
* @return This object for daisy chain support
|
||||||
|
*/
|
||||||
|
public LegacyMappingRuleToJson setAppNameMappingRules(
|
||||||
|
Collection<String> rules) {
|
||||||
|
if (rules != null) {
|
||||||
|
applicationNameMappingRules = rules;
|
||||||
|
} else {
|
||||||
|
applicationNameMappingRules = new ArrayList<>();
|
||||||
|
}
|
||||||
|
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This method will do the conversion based on the already set mapping rules.
|
||||||
|
* First the rules to be converted must be set via setAppNameMappingRules and
|
||||||
|
* setUserGroupMappingRules methods.
|
||||||
|
* @return JSON Format of the provided mapping rules, null if no rules are set
|
||||||
|
*/
|
||||||
|
public String convert() {
|
||||||
|
if (userGroupMappingRules == null && applicationNameMappingRules == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
//creating the basic JSON config structure
|
||||||
|
ObjectNode rootNode = objectMapper.createObjectNode();
|
||||||
|
ArrayNode rulesNode = objectMapper.createArrayNode();
|
||||||
|
rootNode.set("rules", rulesNode);
|
||||||
|
|
||||||
|
//Processing and adding all the user group mapping rules
|
||||||
|
for (String rule : userGroupMappingRules) {
|
||||||
|
rulesNode.add(convertUserGroupMappingRule(rule));
|
||||||
|
}
|
||||||
|
|
||||||
|
//Processing and adding all the application name mapping rules
|
||||||
|
for (String rule : applicationNameMappingRules) {
|
||||||
|
rulesNode.add(convertAppNameMappingRule(rule));
|
||||||
|
}
|
||||||
|
|
||||||
|
//If there are no converted rules we return null
|
||||||
|
if (rulesNode.size() == 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
return objectMapper
|
||||||
|
.writerWithDefaultPrettyPrinter()
|
||||||
|
.writeValueAsString(rootNode);
|
||||||
|
} catch (JsonProcessingException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This intermediate helper method is used to process User Group mapping rules
|
||||||
|
* and invoke the proper mapping rule creation method.
|
||||||
|
* @param rule The legacy format of the single rule to be converted.
|
||||||
|
* @return The ObjectNode which can be added to the rules part of the config.
|
||||||
|
*/
|
||||||
|
ObjectNode convertUserGroupMappingRule(String rule) {
|
||||||
|
String[] mapping = splitRule(rule, 3);
|
||||||
|
String ruleType = mapping[0];
|
||||||
|
String ruleMatch = mapping[1];
|
||||||
|
String ruleTarget = mapping[2];
|
||||||
|
|
||||||
|
if (ruleType.equals(PREFIX_USER_MAPPING)) {
|
||||||
|
return createUserMappingRule(ruleMatch, ruleTarget);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ruleType.equals(PREFIX_GROUP_MAPPING)) {
|
||||||
|
return createGroupMappingRule(ruleMatch, ruleTarget);
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new IllegalArgumentException(
|
||||||
|
"User group mapping rule must start with prefix '" +
|
||||||
|
PREFIX_USER_MAPPING + "' or '" + PREFIX_GROUP_MAPPING + "'");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This intermediate helper method is used to process Application name mapping
|
||||||
|
* rules and invoke the proper mapping rule creation method.
|
||||||
|
* @param rule The legacy format of the single rule to be converted.
|
||||||
|
* @return The ObjectNode which can be added to the rules part of the config.
|
||||||
|
*/
|
||||||
|
ObjectNode convertAppNameMappingRule(String rule) {
|
||||||
|
String[] mapping = splitRule(rule, 2);
|
||||||
|
String ruleMatch = mapping[0];
|
||||||
|
String ruleTarget = mapping[1];
|
||||||
|
|
||||||
|
return createApplicationNameMappingRule(ruleMatch, ruleTarget);
|
||||||
|
}
|
||||||
|
/**
|
||||||
|
* Helper method which splits the rules into parts, and checks if it has
|
||||||
|
* exactly the required amount of parts, and none of them is empty!
|
||||||
|
* @param rule The mapping rule to be split
|
||||||
|
* @param expectedParts The number of expected parts
|
||||||
|
* @return The split String[] of the parts
|
||||||
|
* @throws IllegalArgumentException if the number of parts don't match or any
|
||||||
|
* of them is empty.
|
||||||
|
*/
|
||||||
|
private String[] splitRule(String rule, int expectedParts) {
|
||||||
|
//Splitting
|
||||||
|
String[] mapping = StringUtils
|
||||||
|
.getTrimmedStringCollection(rule, RULE_PART_DELIMITER)
|
||||||
|
.toArray(new String[] {});
|
||||||
|
|
||||||
|
//Checking for part count
|
||||||
|
if (mapping.length != expectedParts) {
|
||||||
|
throw new IllegalArgumentException("Invalid rule '" + rule +
|
||||||
|
"' expected parts: " + expectedParts +
|
||||||
|
" actual parts: " + mapping.length);
|
||||||
|
}
|
||||||
|
|
||||||
|
//Checking for empty parts
|
||||||
|
for (int i = 0; i < mapping.length; i++) {
|
||||||
|
if (mapping[i].length() == 0) {
|
||||||
|
throw new IllegalArgumentException("Invalid rule '" + rule +
|
||||||
|
"' with empty part, mapping rules must not contain empty parts!");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return mapping;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This helper method is to create a default rule node for the converter,
|
||||||
|
* setting fields which are common in all rules.
|
||||||
|
* @param type The type of the rule can be user/group/application
|
||||||
|
* @return The object node with the preset fields
|
||||||
|
*/
|
||||||
|
private ObjectNode createDefaultRuleNode(String type) {
|
||||||
|
return objectMapper
|
||||||
|
.createObjectNode()
|
||||||
|
.put("type", type)
|
||||||
|
//All legacy rule fallback to place to default
|
||||||
|
.put("fallbackResult", "placeDefault")
|
||||||
|
//All legacy rules allow creation
|
||||||
|
.put("create", true);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This method will create the JSON node for a single User Mapping Rule.
|
||||||
|
* @param match The match part of the rule it can be either an actual user
|
||||||
|
* name or '%user' to match all users
|
||||||
|
* @param target The queue to place to user into, some queue path variables
|
||||||
|
* are supported (%user, %primary_group, %secondary_group).
|
||||||
|
* @return The ObjectNode which represents the rule
|
||||||
|
*/
|
||||||
|
private ObjectNode createUserMappingRule(String match, String target) {
|
||||||
|
ObjectNode ruleNode = createDefaultRuleNode("user");
|
||||||
|
MappingQueuePath targetPath = new MappingQueuePath(target);
|
||||||
|
|
||||||
|
//We have a special token in the JSON format to match all user, replacing
|
||||||
|
//matcher
|
||||||
|
if (match.equals(MATCHER_USER)) {
|
||||||
|
match = JSON_MATCH_ALL;
|
||||||
|
}
|
||||||
|
ruleNode.put(JSON_NODE_MATCHES, match);
|
||||||
|
|
||||||
|
switch (targetPath.getLeafName()) {
|
||||||
|
case MAPPING_USER:
|
||||||
|
ruleNode.put(JSON_NODE_POLICY, "user");
|
||||||
|
if (targetPath.hasParent()) {
|
||||||
|
//Parsing parent path, to be able to determine the short name of parent
|
||||||
|
MappingQueuePath targetParentPath =
|
||||||
|
new MappingQueuePath(targetPath.getParent());
|
||||||
|
String parentShortName = targetParentPath.getLeafName();
|
||||||
|
|
||||||
|
if (parentShortName.equals(MAPPING_PRIMARY_GROUP)) {
|
||||||
|
//%primary_group.%user mapping
|
||||||
|
ruleNode.put(JSON_NODE_POLICY, "primaryGroupUser");
|
||||||
|
|
||||||
|
//Yep, this is confusing. The policy primaryGroupUser actually
|
||||||
|
// appends the %primary_group.%user to the parent path, so we need to
|
||||||
|
// remove it from the parent path to avoid duplication.
|
||||||
|
targetPath = new MappingQueuePath(targetParentPath.getParent(),
|
||||||
|
targetPath.getLeafName());
|
||||||
|
} else if (parentShortName.equals(MAPPING_SECONDARY_GROUP)) {
|
||||||
|
//%secondary_group.%user mapping
|
||||||
|
ruleNode.put(JSON_NODE_POLICY, "secondaryGroupUser");
|
||||||
|
|
||||||
|
//Yep, this is confusing. The policy secondaryGroupUser actually
|
||||||
|
// appends the %secondary_group.%user to the parent path, so we need
|
||||||
|
// to remove it from the parent path to avoid duplication.
|
||||||
|
targetPath = new MappingQueuePath(targetParentPath.getParent(),
|
||||||
|
targetPath.getLeafName());
|
||||||
|
}
|
||||||
|
|
||||||
|
//[parent].%user mapping
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
case MAPPING_PRIMARY_GROUP:
|
||||||
|
//[parent].%primary_group mapping
|
||||||
|
ruleNode.put(JSON_NODE_POLICY, "primaryGroup");
|
||||||
|
break;
|
||||||
|
case MAPPING_SECONDARY_GROUP:
|
||||||
|
//[parent].%secondary_group mapping
|
||||||
|
ruleNode.put(JSON_NODE_POLICY, "secondaryGroup");
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
//static path mapping
|
||||||
|
ruleNode.put(JSON_NODE_POLICY, "custom");
|
||||||
|
ruleNode.put(JSON_NODE_CUSTOM_PLACEMENT, targetPath.getFullPath());
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
//if the target queue has a parent part, and the rule can have a parent
|
||||||
|
//we add it to the node
|
||||||
|
if (targetPath.hasParent()) {
|
||||||
|
ruleNode.put(JSON_NODE_PARENT_QUEUE, targetPath.getParent());
|
||||||
|
}
|
||||||
|
|
||||||
|
return ruleNode;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This method will create the JSON node for a single Group Mapping Rule.
|
||||||
|
* @param match The name of the group to match for
|
||||||
|
* @param target The queue to place to user into, some queue path variables
|
||||||
|
* are supported (%user).
|
||||||
|
* @return The ObjectNode which represents the rule
|
||||||
|
*/
|
||||||
|
private ObjectNode createGroupMappingRule(String match, String target) {
|
||||||
|
ObjectNode ruleNode = createDefaultRuleNode("group");
|
||||||
|
MappingQueuePath targetPath = new MappingQueuePath(target);
|
||||||
|
|
||||||
|
//we simply used the source match part all valid legacy matchers are valid
|
||||||
|
//matchers for the JSON format as well
|
||||||
|
ruleNode.put(JSON_NODE_MATCHES, match);
|
||||||
|
|
||||||
|
if (targetPath.getLeafName().matches(MATCHER_USER)) {
|
||||||
|
//g:group:[parent].%user mapping
|
||||||
|
ruleNode.put(JSON_NODE_POLICY, "user");
|
||||||
|
|
||||||
|
//if the target queue has a parent part we add it to the node
|
||||||
|
if (targetPath.hasParent()) {
|
||||||
|
ruleNode.put(JSON_NODE_PARENT_QUEUE, targetPath.getParent());
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
//static path mapping
|
||||||
|
ruleNode.put(JSON_NODE_POLICY, "custom");
|
||||||
|
ruleNode.put(JSON_NODE_CUSTOM_PLACEMENT, targetPath.getFullPath());
|
||||||
|
}
|
||||||
|
|
||||||
|
return ruleNode;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This method will create the JSON node for a single Application Name
|
||||||
|
* Mapping Rule.
|
||||||
|
* @param match The name of the application to match for or %application to
|
||||||
|
* match all applications
|
||||||
|
* @param target The queue to place to user into, some queue path variables
|
||||||
|
* are supported (%application).
|
||||||
|
* @return The ObjectNode which represents the rule
|
||||||
|
*/
|
||||||
|
private ObjectNode createApplicationNameMappingRule(
|
||||||
|
String match, String target) {
|
||||||
|
ObjectNode ruleNode = createDefaultRuleNode("application");
|
||||||
|
MappingQueuePath targetPath = new MappingQueuePath(target);
|
||||||
|
|
||||||
|
//we simply used the source match part all valid legacy matchers are valid
|
||||||
|
//matchers for the JSON format as well
|
||||||
|
ruleNode.put(JSON_NODE_MATCHES, match);
|
||||||
|
|
||||||
|
if (targetPath.getLeafName().matches(MATCHER_APPLICATION)) {
|
||||||
|
//[parent].%application mapping
|
||||||
|
ruleNode.put(JSON_NODE_POLICY, "applicationName");
|
||||||
|
|
||||||
|
//if the target queue has a parent part we add it to the node
|
||||||
|
if (targetPath.hasParent()) {
|
||||||
|
ruleNode.put(JSON_NODE_PARENT_QUEUE, targetPath.getParent());
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
//static path mapping
|
||||||
|
ruleNode.put(JSON_NODE_POLICY, "custom");
|
||||||
|
ruleNode.put(JSON_NODE_CUSTOM_PLACEMENT, targetPath.getFullPath());
|
||||||
|
}
|
||||||
|
|
||||||
|
return ruleNode;
|
||||||
|
}
|
||||||
|
}
|
@ -467,7 +467,7 @@ void assertConfigTestResult(List<MappingRule> rules) {
|
|||||||
assertTrue("Rule's match value should be bob",
|
assertTrue("Rule's match value should be bob",
|
||||||
ruleStr.contains("value='bob'"));
|
ruleStr.contains("value='bob'"));
|
||||||
assertTrue("Rule's action should be place to queue", ruleStr.contains(
|
assertTrue("Rule's action should be place to queue", ruleStr.contains(
|
||||||
"action=PlaceToQueueAction{queueName='%primary_group'}"));
|
"action=PlaceToQueueAction{queueName='%primary_group'"));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -166,7 +166,8 @@ public void testToStrings() {
|
|||||||
"%var", "value");
|
"%var", "value");
|
||||||
MappingRuleAction reject = new MappingRuleActions.RejectAction();
|
MappingRuleAction reject = new MappingRuleActions.RejectAction();
|
||||||
|
|
||||||
assertEquals("PlaceToQueueAction{queueName='queue'}", place.toString());
|
assertEquals("PlaceToQueueAction{queueName='queue',allowCreate=true}",
|
||||||
|
place.toString());
|
||||||
assertEquals("VariableUpdateAction{variableName='%var'" +
|
assertEquals("VariableUpdateAction{variableName='%var'" +
|
||||||
", variableValue='value'}", varUpdate.toString());
|
", variableValue='value'}", varUpdate.toString());
|
||||||
assertEquals("RejectAction", reject.toString());
|
assertEquals("RejectAction", reject.toString());
|
||||||
|
@ -0,0 +1,240 @@
|
|||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.converter;
|
||||||
|
|
||||||
|
import static org.junit.Assert.*;
|
||||||
|
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRule;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public class TestLegacyMappingRuleToJson {
|
||||||
|
|
||||||
|
void validateConversion(String legacyUserGroup, String legacyAppName)
|
||||||
|
throws IOException {
|
||||||
|
//Creating a capacity scheduler config, because this way we can run
|
||||||
|
//both the legacy and the JSON rules through the parser engine, and
|
||||||
|
//we can check if we get the same mapping rules
|
||||||
|
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
|
||||||
|
|
||||||
|
//First we configure the capacity scheduler to parse the legacy config
|
||||||
|
conf.set(
|
||||||
|
CapacitySchedulerConfiguration.MAPPING_RULE_FORMAT,
|
||||||
|
CapacitySchedulerConfiguration.MAPPING_RULE_FORMAT_LEGACY);
|
||||||
|
conf.set(CapacitySchedulerConfiguration.QUEUE_MAPPING, legacyUserGroup);
|
||||||
|
conf.set(CapacitySchedulerConfiguration.QUEUE_MAPPING_NAME, legacyAppName);
|
||||||
|
|
||||||
|
//These are the legacyRules generated by CS, this can be used as a reference
|
||||||
|
//we can test the JSON format against these
|
||||||
|
List<MappingRule> legacyRules = conf.getMappingRules();
|
||||||
|
|
||||||
|
//Converting the legacy format to JSON
|
||||||
|
LegacyMappingRuleToJson converter = new LegacyMappingRuleToJson();
|
||||||
|
String json = converter
|
||||||
|
.setUserGroupMappingRules(legacyUserGroup)
|
||||||
|
.setAppNameMappingRules(legacyAppName)
|
||||||
|
.convert();
|
||||||
|
|
||||||
|
//First we configure the capacity scheduler to parse the CONVERTED JSON
|
||||||
|
conf.set(
|
||||||
|
CapacitySchedulerConfiguration.MAPPING_RULE_FORMAT,
|
||||||
|
CapacitySchedulerConfiguration.MAPPING_RULE_FORMAT_JSON);
|
||||||
|
conf.set(CapacitySchedulerConfiguration.MAPPING_RULE_JSON, json);
|
||||||
|
|
||||||
|
//These are the rules which are generated from the JSON format
|
||||||
|
List<MappingRule> jsonRules = conf.getMappingRules();
|
||||||
|
|
||||||
|
//Sanity check
|
||||||
|
assertEquals("Number of rules should mach",
|
||||||
|
legacyRules.size(), jsonRules.size());
|
||||||
|
|
||||||
|
//We expect ALL rules to match no matter if it was parsed from legacy format
|
||||||
|
//or from JSON
|
||||||
|
for (int i = 0; i < legacyRules.size(); i++) {
|
||||||
|
assertEquals(
|
||||||
|
"Rule #" + i + " should match",
|
||||||
|
legacyRules.get(i).toString(),
|
||||||
|
jsonRules.get(i).toString());
|
||||||
|
|
||||||
|
assertEquals(
|
||||||
|
"Rule #" + i + " fallback should match",
|
||||||
|
legacyRules.get(i).getFallback().toString(),
|
||||||
|
jsonRules.get(i).getFallback().toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testApplicationNameMappingConversion() throws IOException {
|
||||||
|
String appMapping = String.join(",",
|
||||||
|
"namedMatch:simple",
|
||||||
|
"namedMatch:root.deep",
|
||||||
|
"namedMatch:%application",
|
||||||
|
"namedMatch:root.deep.%application",
|
||||||
|
"%application:simple",
|
||||||
|
"%application:root.deep",
|
||||||
|
"%application:%application",
|
||||||
|
"%application:root.deep.%application");
|
||||||
|
|
||||||
|
validateConversion("", appMapping);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGroupMappingConversion() throws IOException {
|
||||||
|
String groupMapping = String.join(",",
|
||||||
|
"g:testers:simple",
|
||||||
|
"g:developers:root.very.deep",
|
||||||
|
"g:users:%user",
|
||||||
|
"g:testers:root.very.deep.%user");
|
||||||
|
|
||||||
|
validateConversion(groupMapping, "");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testUserMappingConversion() throws IOException {
|
||||||
|
String groupMapping = String.join(",",
|
||||||
|
"u:alice:alice",
|
||||||
|
"u:beatrix:root.beatrix",
|
||||||
|
"u:claire:%primary_group",
|
||||||
|
"u:donna:root.deep.%primary_group",
|
||||||
|
"u:emily:%secondary_group",
|
||||||
|
"u:felicity:root.deep.%secondary_group",
|
||||||
|
"u:%user:simple",
|
||||||
|
"u:%user:root.deep",
|
||||||
|
"u:%user:%primary_group",
|
||||||
|
"u:%user:%secondary_group",
|
||||||
|
"u:%user:root.deep.%primary_group",
|
||||||
|
"u:%user:root.deep.%secondary_group",
|
||||||
|
"u:%user:%primary_group.%user",
|
||||||
|
"u:%user:root.%primary_group.%user",
|
||||||
|
"u:%user:root.deep.%primary_group.%user",
|
||||||
|
"u:%user:%secondary_group.%user",
|
||||||
|
"u:%user:root.%secondary_group.%user",
|
||||||
|
"u:%user:root.deep.%secondary_group.%user",
|
||||||
|
"u:%user:%user",
|
||||||
|
"u:%user:root.deep.%user");
|
||||||
|
|
||||||
|
validateConversion(groupMapping, "");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testTotalConversion() throws IOException {
|
||||||
|
String appMapping = String.join(",",
|
||||||
|
"namedMatch:simple",
|
||||||
|
"namedMatch:root.deep",
|
||||||
|
"namedMatch:%application",
|
||||||
|
"namedMatch:root.deep.%application",
|
||||||
|
"%application:simple",
|
||||||
|
"%application:root.deep",
|
||||||
|
"%application:%application",
|
||||||
|
"%application:root.deep.%application");
|
||||||
|
|
||||||
|
String userGroupMapping = String.join(",",
|
||||||
|
"u:alice:alice",
|
||||||
|
"u:beatrix:root.beatrix",
|
||||||
|
"u:claire:%primary_group",
|
||||||
|
"u:donna:root.deep.%primary_group",
|
||||||
|
"u:emily:%secondary_group",
|
||||||
|
"u:felicity:root.deep.%secondary_group",
|
||||||
|
"u:%user:simple",
|
||||||
|
"u:%user:root.deep",
|
||||||
|
"g:testers:simple",
|
||||||
|
"g:developers:root.very.deep",
|
||||||
|
"g:users:%user",
|
||||||
|
"g:testers:root.very.deep.%user",
|
||||||
|
"u:%user:%primary_group",
|
||||||
|
"u:%user:%secondary_group",
|
||||||
|
"u:%user:root.deep.%primary_group",
|
||||||
|
"u:%user:root.deep.%secondary_group",
|
||||||
|
"u:%user:%primary_group.%user",
|
||||||
|
"u:%user:root.%primary_group.%user",
|
||||||
|
"u:%user:root.deep.%primary_group.%user",
|
||||||
|
"u:%user:%secondary_group.%user",
|
||||||
|
"u:%user:root.%secondary_group.%user",
|
||||||
|
"u:%user:root.deep.%secondary_group.%user",
|
||||||
|
"u:%user:%user",
|
||||||
|
"u:%user:root.%user.something",
|
||||||
|
"u:%user:root.deep.%user");
|
||||||
|
|
||||||
|
validateConversion(userGroupMapping, appMapping);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testErrorHandling() {
|
||||||
|
LegacyMappingRuleToJson converter = new LegacyMappingRuleToJson();
|
||||||
|
//Empty converter should return null
|
||||||
|
assertNull(converter.convert());
|
||||||
|
|
||||||
|
converter
|
||||||
|
.setAppNameMappingRules("")
|
||||||
|
.setUserGroupMappingRules("");
|
||||||
|
//Empty converter should still return null
|
||||||
|
assertNull(converter.convert());
|
||||||
|
|
||||||
|
converter
|
||||||
|
.setAppNameMappingRules((Collection<String>)null)
|
||||||
|
.setUserGroupMappingRules((Collection<String>)null);
|
||||||
|
//Setting nulls should also result in null return.
|
||||||
|
assertNull(converter.convert());
|
||||||
|
|
||||||
|
try {
|
||||||
|
converter
|
||||||
|
.setAppNameMappingRules("%application:")
|
||||||
|
.setUserGroupMappingRules("")
|
||||||
|
.convert();
|
||||||
|
fail("Empty app name mapping part should throw exception");
|
||||||
|
} catch (IllegalArgumentException e) {}
|
||||||
|
|
||||||
|
try {
|
||||||
|
converter
|
||||||
|
.setAppNameMappingRules("%application:sdfsdf:sdfsfd")
|
||||||
|
.setUserGroupMappingRules("")
|
||||||
|
.convert();
|
||||||
|
fail("Incorrect number of app name mapping parts should throw exception");
|
||||||
|
} catch (IllegalArgumentException e) {}
|
||||||
|
|
||||||
|
try {
|
||||||
|
converter
|
||||||
|
.setAppNameMappingRules("")
|
||||||
|
.setUserGroupMappingRules("u::root.default")
|
||||||
|
.convert();
|
||||||
|
fail("Empty user group mapping part should throw exception");
|
||||||
|
} catch (IllegalArgumentException e) {}
|
||||||
|
|
||||||
|
try {
|
||||||
|
converter
|
||||||
|
.setAppNameMappingRules("")
|
||||||
|
.setUserGroupMappingRules("u:bob")
|
||||||
|
.convert();
|
||||||
|
fail("Incorrect number of user group mapping parts should " +
|
||||||
|
"throw exception");
|
||||||
|
} catch (IllegalArgumentException e) {}
|
||||||
|
|
||||||
|
try {
|
||||||
|
converter
|
||||||
|
.setAppNameMappingRules("")
|
||||||
|
.setUserGroupMappingRules("X:bob:root.bob")
|
||||||
|
.convert();
|
||||||
|
fail("Invalid user group mapping prefix should throw exception");
|
||||||
|
} catch (IllegalArgumentException e) {}
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in New Issue
Block a user