big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-9593. Support Comma in the value of Scheduler Configuration Mutation API.

Browse Source
This commit is contained in:
tanu.ajmera 2020-02-20 15:24:07 +05:30 committed by Prabhu Joseph
parent d608e94f92
commit b5df1da3a1
2 changed files with 53 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
View File

@ -79,6 +79,7 @@ public class SchedConfCLI extends Configured implements Tool {
private static final String GET_SCHEDULER_CONF = "getConf";
private static final String FORMAT_CONF = "formatConfig";
private static final String HELP_CMD = "help";
private static final String SPLIT_BY_SLASH_COMMA = "(?<!\\\\)\\,";
private static final String CONF_ERR_MSG = "Specify configuration key " +
"value as confKey=confVal.";
@ -410,7 +411,8 @@ void globalUpdates(String args, SchedConfUpdateInfo updateInfo) {
return;
}
HashMap<String, String> globalUpdates = new HashMap<>();
for (String globalUpdate : args.split(",")) {
for (String globalUpdate : args.split(SPLIT_BY_SLASH_COMMA)) {
globalUpdate = globalUpdate.replace("\\", "");
putKeyValuePair(globalUpdates, globalUpdate);
}
updateInfo.setGlobalParams(globalUpdates);
@ -421,8 +423,9 @@ private QueueConfigInfo getQueueConfigInfo(String arg) {
String queuePath = args[0];
Map<String, String> queueConfigs = new HashMap<>();
if (args.length > 1) {
String[] queueArgs = args[1].split(",");
String[] queueArgs = args[1].split(SPLIT_BY_SLASH_COMMA);
for (int i = 0; i < queueArgs.length; ++i) {
queueArgs[i] = queueArgs[i].replace("\\", "");
putKeyValuePair(queueConfigs, queueArgs[i]);
}
}
@ -461,13 +464,22 @@ private void printUsage() {
+ "Example (adding queues): yarn schedulerconf -add "
+ "\"root.a.a1:capacity=100,maximum-capacity=100;root.a.a2:capacity=0,"
+ "maximum-capacity=0\"\n"
+ "Example (adding queues with comma in value): yarn schedulerconf "
+ "-add \"root.default:acl_administer_queue=user1\\,user2 group1\\,"
+ "group2,maximum-capacity=100;root.a.a2:capacity=0\"\n"
+ "Example (removing queues): yarn schedulerconf -remove \"root.a.a1;"
+ "root.a.a2\"\n"
+ "Example (updating queues): yarn schedulerconf -update \"root.a.a1"
+ ":capacity=25,maximum-capacity=25;root.a.a2:capacity=75,"
+ "maximum-capacity=75\"\n"
+ "Example (updating queues with comma in value): yarn schedulerconf "
+ "-update \"root.default:acl_administer_queue=user1\\,user2 group1\\,"
+ "group2,maximum-capacity=25;root.a.a2:capacity=75\"\n"
+ "Example (global scheduler update): yarn schedulerconf "
+ "-global yarn.scheduler.capacity.maximum-applications=10000\n"
+ "Example (global scheduler update with comma in value): yarn "
+ "schedulerconf "
+ "-global \"acl_administer_queue=user1\\,user2 group1\\,group2\"\n"
+ "Example (format scheduler configuration): yarn schedulerconf "
+ "-format\n"
+ "Example (get scheduler configuration): yarn schedulerconf "

39
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
View File

@ -315,6 +315,19 @@ public void testAddQueues() {
assertEquals("cVal1", cParams.get("c1"));
}
@Test(timeout = 10000)
public void testAddQueuesWithCommaInValue() {
SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo();
cli.addQueues("root.a:a1=a1Val1\\,a1Val2 a1Val3,a2=a2Val1\\,a2Val2",
schedUpdateInfo);
QueueConfigInfo addInfo = schedUpdateInfo.getAddQueueInfo().get(0);
assertEquals("root.a", addInfo.getQueue());
Map<String, String> params = addInfo.getParams();
assertEquals(2, params.size());
assertEquals("a1Val1,a1Val2 a1Val3", params.get("a1"));
assertEquals("a2Val1,a2Val2", params.get("a2"));
}
@Test(timeout = 10000)
public void testRemoveQueues() {
SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo();
@ -353,6 +366,19 @@ public void testUpdateQueues() {
assertEquals("cVal1", cParams.get("c1"));
}
@Test(timeout = 10000)
public void testUpdateQueuesWithCommaInValue() {
SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo();
cli.updateQueues("root.a:a1=a1Val1\\,a1Val2 a1Val3,a2=a2Val1\\,a2Val2",
schedUpdateInfo);
QueueConfigInfo updateInfo = schedUpdateInfo.getUpdateQueueInfo().get(0);
assertEquals("root.a", updateInfo.getQueue());
Map<String, String> params = updateInfo.getParams();
assertEquals(2, params.size());
assertEquals("a1Val1,a1Val2 a1Val3", params.get("a1"));
assertEquals("a2Val1,a2Val2", params.get("a2"));
}
@Test(timeout = 10000)
public void testGlobalUpdate() {
SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo();
@ -363,4 +389,17 @@ public void testGlobalUpdate() {
assertEquals("schedVal1", globalInfo.get("schedKey1"));
assertEquals("schedVal2", globalInfo.get("schedKey2"));
}
@Test(timeout = 10000)
public void testGlobalUpdateWithCommaInValue() {
SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo();
cli.globalUpdates(
"schedKey1=schedVal1.1\\,schedVal1.2 schedVal1.3,schedKey2=schedVal2",
schedUpdateInfo);
Map<String, String> globalInfo = schedUpdateInfo.getGlobalParams();
assertEquals(2, globalInfo.size());
assertEquals("schedVal1.1,schedVal1.2 schedVal1.3",
globalInfo.get("schedKey1"));
assertEquals("schedVal2", globalInfo.get("schedKey2"));
}
}