big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-9995. Code cleanup in TestSchedConfCLI. Contributed by Bilwa S T.

Browse Source
This commit is contained in:
Szilard Nemeth 2020-04-09 15:46:25 +02:00
parent 6fb29d0f17
commit c04a484aed
1 changed files with 86 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

176
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
View File

@ -66,6 +66,7 @@
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNull; import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
@ -74,17 +75,10 @@
*/ */
public class TestSchedConfCLI extends JerseyTestBase { public class TestSchedConfCLI extends JerseyTestBase {
private ByteArrayOutputStream sysOutStream;
private PrintStream sysOut;
private ByteArrayOutputStream sysErrStream;
private PrintStream sysErr;
private SchedConfCLI cli; private SchedConfCLI cli;
private static MockRM rm; private static MockRM rm;
private static String userName; private static String userName;
private static CapacitySchedulerConfiguration csConf;
private static final File CONF_FILE = new File(new File("target", private static final File CONF_FILE = new File(new File("target",
"test-classes"), YarnConfiguration.CS_CONFIGURATION_FILE); "test-classes"), YarnConfiguration.CS_CONFIGURATION_FILE);
@ -101,14 +95,6 @@ public TestSchedConfCLI() {
@Before @Before
public void setUp() { public void setUp() {
sysOutStream = new ByteArrayOutputStream();
sysOut = new PrintStream(sysOutStream);
System.setOut(sysOut);
sysErrStream = new ByteArrayOutputStream();
sysErr = new PrintStream(sysErrStream);
System.setErr(sysErr);
cli = new SchedConfCLI(); cli = new SchedConfCLI();
} }
@ -131,8 +117,8 @@ protected void configureServlets() {
+ ioe.getMessage(), ioe); + ioe.getMessage(), ioe);
} }
csConf = new CapacitySchedulerConfiguration(new Configuration(false), CapacitySchedulerConfiguration csConf = new
false); CapacitySchedulerConfiguration(new Configuration(false), false);
setupQueueConfiguration(csConf); setupQueueConfiguration(csConf);
try { try {
@ -220,6 +206,9 @@ private void cleanUp() throws Exception {
@Test(timeout = 10000) @Test(timeout = 10000)
public void testGetSchedulerConf() throws Exception { public void testGetSchedulerConf() throws Exception {
ByteArrayOutputStream sysOutStream = new ByteArrayOutputStream();
PrintStream sysOut = new PrintStream(sysOutStream);
System.setOut(sysOut);
try { try {
super.setUp(); super.setUp();
GuiceServletConfig.setInjector( GuiceServletConfig.setInjector(
@ -268,51 +257,46 @@ public void testFormatSchedulerConf() throws Exception {
@Test(timeout = 10000) @Test(timeout = 10000)
public void testInvalidConf() throws Exception { public void testInvalidConf() throws Exception {
// conf pair with no key should be invalid ByteArrayOutputStream sysErrStream = new ByteArrayOutputStream();
int exitCode = cli.run(new String[] {"-add", "root.a:=confVal"}); PrintStream sysErr = new PrintStream(sysErrStream);
assertTrue("Should return an error code", exitCode != 0); System.setErr(sysErr);
assertTrue(sysErrStream.toString().contains("Specify configuration key " +
"value as confKey=confVal."));
exitCode = cli.run(new String[] {"-update", "root.a:=confVal"});
assertTrue("Should return an error code", exitCode != 0);
assertTrue(sysErrStream.toString().contains("Specify configuration key " +
"value as confKey=confVal."));
exitCode = cli.run(new String[] {"-add", "root.a:confKey=confVal=conf"}); // conf pair with no key should be invalid
assertTrue("Should return an error code", exitCode != 0); executeCommand(sysErrStream, "-add", "root.a:=confVal");
assertTrue(sysErrStream.toString().contains("Specify configuration key " + executeCommand(sysErrStream, "-update", "root.a:=confVal");
"value as confKey=confVal.")); executeCommand(sysErrStream, "-add", "root.a:confKey=confVal=conf");
exitCode = cli.run(new String[] {"-update", "root.a:confKey=confVal=c"}); executeCommand(sysErrStream, "-update", "root.a:confKey=confVal=c");
assertTrue("Should return an error code", exitCode != 0); }
assertTrue(sysErrStream.toString().contains("Specify configuration key " +
"value as confKey=confVal.")); private void executeCommand(ByteArrayOutputStream sysErrStream, String op,
String queueConf) throws Exception {
int exitCode = cli.run(new String[] {op, queueConf});
assertNotEquals("Should return an error code", 0, exitCode);
assertTrue(sysErrStream.toString()
.contains("Specify configuration key " + "value as confKey=confVal."));
} }
@Test(timeout = 10000) @Test(timeout = 10000)
public void testAddQueues() { public void testAddQueues() {
SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo(); SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo();
cli.addQueues("root.a:a1=aVal1,a2=aVal2,a3=", schedUpdateInfo); cli.addQueues("root.a:a1=aVal1,a2=aVal2,a3=", schedUpdateInfo);
QueueConfigInfo addInfo = schedUpdateInfo.getAddQueueInfo().get(0); Map<String, String> paramValues = new HashMap<>();
assertEquals("root.a", addInfo.getQueue()); List<QueueConfigInfo> addQueueInfo = schedUpdateInfo.getAddQueueInfo();
Map<String, String> params = addInfo.getParams(); paramValues.put("a1", "aVal1");
assertEquals(3, params.size()); paramValues.put("a2", "aVal2");
assertEquals("aVal1", params.get("a1")); paramValues.put("a3", null);
assertEquals("aVal2", params.get("a2")); validateQueueConfigInfo(addQueueInfo, 0, "root.a", paramValues);
assertNull(params.get("a3"));
schedUpdateInfo = new SchedConfUpdateInfo(); schedUpdateInfo = new SchedConfUpdateInfo();
cli.addQueues("root.b:b1=bVal1;root.c:c1=cVal1", schedUpdateInfo); cli.addQueues("root.b:b1=bVal1;root.c:c1=cVal1", schedUpdateInfo);
assertEquals(2, schedUpdateInfo.getAddQueueInfo().size()); addQueueInfo = schedUpdateInfo.getAddQueueInfo();
QueueConfigInfo bAddInfo = schedUpdateInfo.getAddQueueInfo().get(0); assertEquals(2, addQueueInfo.size());
assertEquals("root.b", bAddInfo.getQueue()); paramValues.clear();
Map<String, String> bParams = bAddInfo.getParams(); paramValues.put("b1", "bVal1");
assertEquals(1, bParams.size()); validateQueueConfigInfo(addQueueInfo, 0, "root.b", paramValues);
assertEquals("bVal1", bParams.get("b1")); paramValues.clear();
QueueConfigInfo cAddInfo = schedUpdateInfo.getAddQueueInfo().get(1); paramValues.put("c1", "cVal1");
assertEquals("root.c", cAddInfo.getQueue()); validateQueueConfigInfo(addQueueInfo, 1, "root.c", paramValues);
Map<String, String> cParams = cAddInfo.getParams();
assertEquals(1, cParams.size());
assertEquals("cVal1", cParams.get("c1"));
} }
@Test(timeout = 10000) @Test(timeout = 10000)
@ -320,12 +304,11 @@ public void testAddQueuesWithCommaInValue() {
SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo(); SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo();
cli.addQueues("root.a:a1=a1Val1\\,a1Val2 a1Val3,a2=a2Val1\\,a2Val2", cli.addQueues("root.a:a1=a1Val1\\,a1Val2 a1Val3,a2=a2Val1\\,a2Val2",
schedUpdateInfo); schedUpdateInfo);
QueueConfigInfo addInfo = schedUpdateInfo.getAddQueueInfo().get(0); List<QueueConfigInfo> addQueueInfo = schedUpdateInfo.getAddQueueInfo();
assertEquals("root.a", addInfo.getQueue()); Map<String, String> params = new HashMap<>();
Map<String, String> params = addInfo.getParams(); params.put("a1", "a1Val1,a1Val2 a1Val3");
assertEquals(2, params.size()); params.put("a2", "a2Val1,a2Val2");
assertEquals("a1Val1,a1Val2 a1Val3", params.get("a1")); validateQueueConfigInfo(addQueueInfo, 0, "root.a", params);
assertEquals("a2Val1,a2Val2", params.get("a2"));
} }
@Test(timeout = 10000) @Test(timeout = 10000)
@ -342,28 +325,35 @@ public void testRemoveQueues() {
@Test(timeout = 10000) @Test(timeout = 10000)
public void testUpdateQueues() { public void testUpdateQueues() {
SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo(); SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo();
Map<String, String> paramValues = new HashMap<>();
cli.updateQueues("root.a:a1=aVal1,a2=aVal2,a3=", schedUpdateInfo); cli.updateQueues("root.a:a1=aVal1,a2=aVal2,a3=", schedUpdateInfo);
QueueConfigInfo updateInfo = schedUpdateInfo.getUpdateQueueInfo().get(0); List<QueueConfigInfo> updateQueueInfo = schedUpdateInfo
assertEquals("root.a", updateInfo.getQueue()); .getUpdateQueueInfo();
Map<String, String> params = updateInfo.getParams(); paramValues.put("a1", "aVal1");
assertEquals(3, params.size()); paramValues.put("a2", "aVal2");
assertEquals("aVal1", params.get("a1")); paramValues.put("a3", null);
assertEquals("aVal2", params.get("a2")); validateQueueConfigInfo(updateQueueInfo, 0, "root.a", paramValues);
assertNull(params.get("a3"));
schedUpdateInfo = new SchedConfUpdateInfo(); schedUpdateInfo = new SchedConfUpdateInfo();
cli.updateQueues("root.b:b1=bVal1;root.c:c1=cVal1", schedUpdateInfo); cli.updateQueues("root.b:b1=bVal1;root.c:c1=cVal1", schedUpdateInfo);
assertEquals(2, schedUpdateInfo.getUpdateQueueInfo().size()); updateQueueInfo = schedUpdateInfo.getUpdateQueueInfo();
QueueConfigInfo bUpdateInfo = schedUpdateInfo.getUpdateQueueInfo().get(0); assertEquals(2, updateQueueInfo.size());
assertEquals("root.b", bUpdateInfo.getQueue()); paramValues.clear();
Map<String, String> bParams = bUpdateInfo.getParams(); paramValues.put("b1", "bVal1");
assertEquals(1, bParams.size()); validateQueueConfigInfo(updateQueueInfo, 0, "root.b", paramValues);
assertEquals("bVal1", bParams.get("b1")); paramValues.clear();
QueueConfigInfo cUpdateInfo = schedUpdateInfo.getUpdateQueueInfo().get(1); paramValues.put("c1", "cVal1");
assertEquals("root.c", cUpdateInfo.getQueue()); validateQueueConfigInfo(updateQueueInfo, 1, "root.c", paramValues);
Map<String, String> cParams = cUpdateInfo.getParams(); }
assertEquals(1, cParams.size());
assertEquals("cVal1", cParams.get("c1")); private void validateQueueConfigInfo(
List<QueueConfigInfo> updateQueueInfo, int index, String queuename,
Map<String, String> paramValues) {
QueueConfigInfo updateInfo = updateQueueInfo.get(index);
assertEquals(queuename, updateInfo.getQueue());
Map<String, String> params = updateInfo.getParams();
assertEquals(paramValues.size(), params.size());
paramValues.forEach((k, v) -> assertEquals(v, params.get(k)));
} }
@Test(timeout = 10000) @Test(timeout = 10000)
@ -371,12 +361,12 @@ public void testUpdateQueuesWithCommaInValue() {
SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo(); SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo();
cli.updateQueues("root.a:a1=a1Val1\\,a1Val2 a1Val3,a2=a2Val1\\,a2Val2", cli.updateQueues("root.a:a1=a1Val1\\,a1Val2 a1Val3,a2=a2Val1\\,a2Val2",
schedUpdateInfo); schedUpdateInfo);
QueueConfigInfo updateInfo = schedUpdateInfo.getUpdateQueueInfo().get(0); List<QueueConfigInfo> updateQueueInfo = schedUpdateInfo
assertEquals("root.a", updateInfo.getQueue()); .getUpdateQueueInfo();
Map<String, String> params = updateInfo.getParams(); Map<String, String> paramValues = new HashMap<>();
assertEquals(2, params.size()); paramValues.put("a1", "a1Val1,a1Val2 a1Val3");
assertEquals("a1Val1,a1Val2 a1Val3", params.get("a1")); paramValues.put("a2", "a2Val1,a2Val2");
assertEquals("a2Val1,a2Val2", params.get("a2")); validateQueueConfigInfo(updateQueueInfo, 0, "root.a", paramValues);
} }
@Test(timeout = 10000) @Test(timeout = 10000)
@ -384,10 +374,10 @@ public void testGlobalUpdate() {
SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo(); SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo();
cli.globalUpdates("schedKey1=schedVal1,schedKey2=schedVal2", cli.globalUpdates("schedKey1=schedVal1,schedKey2=schedVal2",
schedUpdateInfo); schedUpdateInfo);
Map<String, String> globalInfo = schedUpdateInfo.getGlobalParams(); Map<String, String> paramValues = new HashMap<>();
assertEquals(2, globalInfo.size()); paramValues.put("schedKey1", "schedVal1");
assertEquals("schedVal1", globalInfo.get("schedKey1")); paramValues.put("schedKey2", "schedVal2");
assertEquals("schedVal2", globalInfo.get("schedKey2")); validateGlobalParams(schedUpdateInfo, paramValues);
} }
@Test(timeout = 10000) @Test(timeout = 10000)
@ -396,10 +386,16 @@ public void testGlobalUpdateWithCommaInValue() {
cli.globalUpdates( cli.globalUpdates(
"schedKey1=schedVal1.1\\,schedVal1.2 schedVal1.3,schedKey2=schedVal2", "schedKey1=schedVal1.1\\,schedVal1.2 schedVal1.3,schedKey2=schedVal2",
schedUpdateInfo); schedUpdateInfo);
Map<String, String> paramValues = new HashMap<>();
paramValues.put("schedKey1", "schedVal1.1,schedVal1.2 schedVal1.3");
paramValues.put("schedKey2", "schedVal2");
validateGlobalParams(schedUpdateInfo, paramValues);
}
private void validateGlobalParams(SchedConfUpdateInfo schedUpdateInfo,
Map<String, String> paramValues) {
Map<String, String> globalInfo = schedUpdateInfo.getGlobalParams(); Map<String, String> globalInfo = schedUpdateInfo.getGlobalParams();
assertEquals(2, globalInfo.size()); assertEquals(paramValues.size(), globalInfo.size());
assertEquals("schedVal1.1,schedVal1.2 schedVal1.3", paramValues.forEach((k, v) -> assertEquals(v, globalInfo.get(k)));
globalInfo.get("schedKey1"));
assertEquals("schedVal2", globalInfo.get("schedKey2"));
} }
} }