From 7dc272199ffea28c8597151e17d4606928a7f9f3 Mon Sep 17 00:00:00 2001 From: Suma Shivaprasad Date: Wed, 28 Nov 2018 00:42:00 -0800 Subject: [PATCH] YARN-9034. ApplicationCLI should have option to take clusterId. Contributed by Rohith Sharma K S. --- .../yarn/client/cli/ApplicationCLI.java | 17 ++++ .../hadoop/yarn/client/cli/ClusterCLI.java | 2 + .../hadoop/yarn/client/cli/NodeCLI.java | 2 + .../hadoop/yarn/client/cli/QueueCLI.java | 2 +- .../apache/hadoop/yarn/client/cli/TopCLI.java | 1 + .../hadoop/yarn/client/cli/YarnCLI.java | 9 +- .../yarn/client/cli/TestClusterCLI.java | 41 ++++----- .../hadoop/yarn/client/cli/TestYarnCLI.java | 89 +++++++++---------- .../api/impl/TimelineReaderClientImpl.java | 2 + 9 files changed, 92 insertions(+), 73 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index 2b3415413c..ff8142ab79 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.api.records.SignalContainerCommand; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.client.api.AppAdminClient; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException; @@ -113,6 +114,7 @@ public class ApplicationCLI extends YarnCLI { public static final String VERSION = "version"; public static final String STATES = "states"; public static final String SHELL_CMD = "shell"; + public static final String CLUSTER_ID_OPTION = "clusterId"; private static String firstArg = null; @@ -278,6 +280,8 @@ public int run(String[] args) throws Exception { "the ability to finalize the upgrade automatically."); opts.addOption(UPGRADE_CANCEL, false, "Works with -upgrade option to " + "cancel current upgrade."); + opts.addOption(CLUSTER_ID_OPTION, true, "ClusterId. " + + "By default, it will take default cluster id from the RM"); opts.getOption(LAUNCH_CMD).setArgName("Application Name> ()); - ClusterCLI cli = new ClusterCLI(); - cli.setClient(client); - cli.setSysOutPrintStream(sysOut); - cli.setSysErrPrintStream(sysErr); + ClusterCLI cli = createAndGetClusterCLI(); int rc = cli.run(new String[] { ClusterCLI.CMD, "-" + ClusterCLI.LIST_LABELS_CMD }); @@ -154,9 +140,7 @@ public void testGetEmptyClusterNodeLabels() throws Exception { @Test public void testHelp() throws Exception { - ClusterCLI cli = new ClusterCLI(); - cli.setSysOutPrintStream(sysOut); - cli.setSysErrPrintStream(sysErr); + ClusterCLI cli = createAndGetClusterCLI(); int rc = cli.run(new String[] { "cluster", "--help" }); @@ -192,4 +176,15 @@ public void testHelp() throws Exception { pw.close(); verify(sysOut).println(baos.toString("UTF-8")); } + + private ClusterCLI createAndGetClusterCLI() { + ClusterCLI cli = new ClusterCLI() { + @Override protected void createAndStartYarnClient() { + } + }; + cli.setClient(client); + cli.setSysOutPrintStream(sysOut); + cli.setSysErrPrintStream(sysErr); + return cli; + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 672e3d7e43..8c59a6133a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -993,10 +993,7 @@ public void testContainersHelpCommand() throws Exception { @Test (timeout = 5000) public void testNodesHelpCommand() throws Exception { - NodeCLI nodeCLI = new NodeCLI(); - nodeCLI.setClient(client); - nodeCLI.setSysOutPrintStream(sysOut); - nodeCLI.setSysErrPrintStream(sysErr); + NodeCLI nodeCLI = createAndGetNodeCLI(); nodeCLI.run(new String[] {}); Assert.assertEquals(createNodeCLIHelpMessage(), sysOutStream.toString()); @@ -1290,9 +1287,7 @@ public void testListClusterNodes() throws Exception { nodeReports.addAll(getNodeReports(1, NodeState.REBOOTED)); nodeReports.addAll(getNodeReports(1, NodeState.LOST)); - NodeCLI cli = new NodeCLI(); - cli.setClient(client); - cli.setSysOutPrintStream(sysOut); + NodeCLI cli = createAndGetNodeCLI(); Set nodeStates = new HashSet(); nodeStates.add(NodeState.NEW); @@ -1545,12 +1540,9 @@ private List getNodeReports( @Test public void testNodeStatus() throws Exception { NodeId nodeId = NodeId.newInstance("host0", 0); - NodeCLI cli = new NodeCLI(); when(client.getNodeReports()) .thenReturn(getNodeReports(3, NodeState.RUNNING, false, false, false)); - cli.setClient(client); - cli.setSysOutPrintStream(sysOut); - cli.setSysErrPrintStream(sysErr); + NodeCLI cli = createAndGetNodeCLI(); int result = cli.run(new String[] { "-status", nodeId.toString() }); assertEquals(0, result); verify(client).getNodeReports(); @@ -1583,12 +1575,9 @@ public void testNodeStatus() throws Exception { @Test public void testNodeStatusWithEmptyNodeLabels() throws Exception { NodeId nodeId = NodeId.newInstance("host0", 0); - NodeCLI cli = new NodeCLI(); when(client.getNodeReports()).thenReturn( getNodeReports(3, NodeState.RUNNING)); - cli.setClient(client); - cli.setSysOutPrintStream(sysOut); - cli.setSysErrPrintStream(sysErr); + NodeCLI cli = createAndGetNodeCLI(); int result = cli.run(new String[] { "-status", nodeId.toString() }); assertEquals(0, result); verify(client).getNodeReports(); @@ -1620,12 +1609,9 @@ public void testNodeStatusWithEmptyNodeLabels() throws Exception { @Test public void testNodeStatusWithEmptyResourceUtilization() throws Exception { NodeId nodeId = NodeId.newInstance("host0", 0); - NodeCLI cli = new NodeCLI(); when(client.getNodeReports()) .thenReturn(getNodeReports(3, NodeState.RUNNING, false, true, true)); - cli.setClient(client); - cli.setSysOutPrintStream(sysOut); - cli.setSysErrPrintStream(sysErr); + NodeCLI cli = createAndGetNodeCLI(); int result = cli.run(new String[] { "-status", nodeId.toString() }); assertEquals(0, result); verify(client).getNodeReports(); @@ -1657,12 +1643,10 @@ public void testNodeStatusWithEmptyResourceUtilization() throws Exception { @Test public void testAbsentNodeStatus() throws Exception { NodeId nodeId = NodeId.newInstance("Absenthost0", 0); - NodeCLI cli = new NodeCLI(); + when(client.getNodeReports()).thenReturn( getNodeReports(0, NodeState.RUNNING)); - cli.setClient(client); - cli.setSysOutPrintStream(sysOut); - cli.setSysErrPrintStream(sysErr); + NodeCLI cli = createAndGetNodeCLI(); int result = cli.run(new String[] { "-status", nodeId.toString() }); assertEquals(0, result); verify(client).getNodeReports(); @@ -1702,10 +1686,7 @@ public void testMissingArguments() throws Exception { createContainerCLIHelpMessage()), normalize(sysOutStream.toString())); sysOutStream.reset(); - NodeCLI nodeCLI = new NodeCLI(); - nodeCLI.setClient(client); - nodeCLI.setSysOutPrintStream(sysOut); - nodeCLI.setSysErrPrintStream(sysErr); + NodeCLI nodeCLI = createAndGetNodeCLI(); result = nodeCLI.run(new String[] { "-status" }); Assert.assertEquals(result, -1); Assert.assertEquals(String.format("Missing argument for options%n%1s", @@ -1774,10 +1755,7 @@ public void testGetQueueInfoOverrideIntraQueuePreemption() throws Exception { yarnClient.init(yarnConf); yarnClient.start(); - QueueCLI cli = new QueueCLI(); - cli.setClient(yarnClient); - cli.setSysOutPrintStream(sysOut); - cli.setSysErrPrintStream(sysErr); + QueueCLI cli = createAndGetQueueCLI(yarnClient); sysOutStream.reset(); // Get status for the root.a queue int result = cli.run(new String[] { "-status", "a" }); @@ -1788,10 +1766,7 @@ public void testGetQueueInfoOverrideIntraQueuePreemption() throws Exception { // In-queue preemption is disabled at the "root.a" queue level Assert.assertTrue(queueStatusOut .contains("Intra-queue Preemption : disabled")); - cli = new QueueCLI(); - cli.setClient(yarnClient); - cli.setSysOutPrintStream(sysOut); - cli.setSysErrPrintStream(sysErr); + cli = createAndGetQueueCLI(yarnClient); sysOutStream.reset(); // Get status for the root.a.a1 queue result = cli.run(new String[] { "-status", "a1" }); @@ -1836,10 +1811,7 @@ public void testGetQueueInfoPreemptionEnabled() throws Exception { yarnClient.init(yarnConf); yarnClient.start(); - QueueCLI cli = new QueueCLI(); - cli.setClient(yarnClient); - cli.setSysOutPrintStream(sysOut); - cli.setSysErrPrintStream(sysErr); + QueueCLI cli = createAndGetQueueCLI(yarnClient); sysOutStream.reset(); int result = cli.run(new String[] { "-status", "a1" }); assertEquals(0, result); @@ -1880,10 +1852,7 @@ public void testGetQueueInfoPreemptionDisabled() throws Exception { yarnClient.init(yarnConf); yarnClient.start(); - QueueCLI cli = new QueueCLI(); - cli.setClient(yarnClient); - cli.setSysOutPrintStream(sysOut); - cli.setSysErrPrintStream(sysErr); + QueueCLI cli = createAndGetQueueCLI(yarnClient); sysOutStream.reset(); int result = cli.run(new String[] { "-status", "a1" }); assertEquals(0, result); @@ -2101,14 +2070,36 @@ private List getNodeReports(int noOfNodes, NodeState state, } private ApplicationCLI createAndGetAppCLI() { - ApplicationCLI cli = new ApplicationCLI(); + ApplicationCLI cli = new ApplicationCLI() { + @Override protected void createAndStartYarnClient() { + } + }; cli.setClient(client); cli.setSysOutPrintStream(sysOut); + cli.setSysErrPrintStream(sysErr); return cli; } - + private QueueCLI createAndGetQueueCLI() { - QueueCLI cli = new QueueCLI(); + return createAndGetQueueCLI(client); + } + + private QueueCLI createAndGetQueueCLI(YarnClient client) { + QueueCLI cli = new QueueCLI() { + @Override protected void createAndStartYarnClient() { + } + }; + cli.setClient(client); + cli.setSysOutPrintStream(sysOut); + cli.setSysErrPrintStream(sysErr); + return cli; + } + + private NodeCLI createAndGetNodeCLI() { + NodeCLI cli = new NodeCLI() { + @Override protected void createAndStartYarnClient() { + } + }; cli.setClient(client); cli.setSysOutPrintStream(sysOut); cli.setSysErrPrintStream(sysErr); @@ -2152,6 +2143,9 @@ private String createApplicationCLIHelpMessage() throws IOException { pw.println(" deprecated, this new command"); pw.println(" 'changeQueue' performs same"); pw.println(" functionality."); + pw.println(" -clusterId ClusterId. By default, it will"); + pw.println(" take default cluster id from the"); + pw.println(" RM"); pw.println(" -component Works with -flex option to"); pw.println(" change the number of"); pw.println(" components/containers running"); @@ -2298,6 +2292,8 @@ private String createApplicationAttemptCLIHelpMessage() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintWriter pw = new PrintWriter(baos); pw.println("usage: applicationattempt"); + pw.println(" -clusterId ClusterId. By default, it will take"); + pw.println(" default cluster id from the RM"); pw.println(" -fail Fails application attempt."); pw.println(" -help Displays help for all commands."); pw.println(" -list List application attempts for"); @@ -2314,6 +2310,7 @@ private String createContainerCLIHelpMessage() throws IOException { PrintWriter pw = new PrintWriter(baos); pw.println("usage: container"); pw.println(" -appTypes Works with -list to specify the app type when application name is provided."); + pw.println(" -clusterId ClusterId. By default, it will take default cluster id from the RM "); pw.println(" -components Works with -list to filter instances based on input comma-separated list of component names."); pw.println(" -help Displays help for all commands."); pw.println(" -list List containers for application attempt when application attempt ID is provided. When application name is provided, then it finds the instances of the application based on app's own implementation, and -appTypes option must be specified unless it is the default yarn-service type. With app name, it supports optional use of -version to filter instances based on app version, -components to filter instances based on component names, -states to filter instances based on instance state."); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java index b138a54af7..db53f93136 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineReaderClientImpl.java @@ -94,6 +94,8 @@ protected void serviceInit(Configuration conf) throws Exception { conf, timelineReaderWebAppAddress, RESOURCE_URI_STR_V2); clusterId = conf.get(YarnConfiguration.RM_CLUSTER_ID, YarnConfiguration.DEFAULT_RM_CLUSTER_ID); + LOG.info("Initialized TimelineReader URI=" + baseUri + ", clusterId=" + + clusterId); super.serviceInit(conf); }