From a0de7028515eebe1c526cc42808cdbc8ed6b4e2a Mon Sep 17 00:00:00 2001 From: Wangda Tan Date: Mon, 21 Dec 2015 11:36:22 -0800 Subject: [PATCH] YARN-4290. Add -showDetails option to YARN Nodes CLI to print all nodes reports information. (Sunil G via wangda) --- hadoop-yarn-project/CHANGES.txt | 3 + .../hadoop/yarn/client/cli/NodeCLI.java | 76 ++++++++++++++++++- .../hadoop/yarn/client/cli/TestYarnCLI.java | 45 +++++++++-- 3 files changed, 115 insertions(+), 9 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index f5750030c3..10f54e6e72 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -634,6 +634,9 @@ Release 2.8.0 - UNRELEASED YARN-4480. Clean up some inappropriate imports. (Kai Zheng via umamahesh) + YARN-4290. Add -showDetails option to YARN Nodes CLI to print all nodes reports + information. (Sunil G via wangda) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java index c93b2dfcae..c112857265 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java @@ -54,6 +54,7 @@ public class NodeCLI extends YarnCLI { private static final String NODE_STATE_CMD = "states"; private static final String NODE_ALL = "all"; + private static final String NODE_SHOW_DETAILS = "showDetails"; public static void main(String[] args) throws Exception { NodeCLI cli = new NodeCLI(); @@ -72,7 +73,8 @@ public class NodeCLI extends YarnCLI { opts.addOption(STATUS_CMD, true, "Prints the status report of the node."); opts.addOption(LIST_CMD, false, "List all running nodes. " + "Supports optional use of -states to filter nodes " + - "based on node state, all -all to list all nodes."); + "based on node state, all -all to list all nodes, " + + "-showDetails to display more details about each node."); Option nodeStateOpt = new Option(NODE_STATE_CMD, true, "Works with -list to filter nodes based on input comma-separated list of node states."); nodeStateOpt.setValueSeparator(','); @@ -82,6 +84,9 @@ public class NodeCLI extends YarnCLI { Option allOpt = new Option(NODE_ALL, false, "Works with -list to list all nodes."); opts.addOption(allOpt); + Option showDetailsOpt = new Option(NODE_SHOW_DETAILS, false, + "Works with -list to show more details about each node."); + opts.addOption(showDetailsOpt); opts.getOption(STATUS_CMD).setArgName("NodeId"); int exitCode = -1; @@ -119,7 +124,13 @@ public class NodeCLI extends YarnCLI { } else { nodeStates.add(NodeState.RUNNING); } - listClusterNodes(nodeStates); + + // List all node details with more information. + if (cliParser.hasOption(NODE_SHOW_DETAILS)) { + listDetailedClusterNodes(nodeStates); + } else { + listClusterNodes(nodeStates); + } } else if (cliParser.hasOption(HELP_CMD)) { printUsage(opts); return 0; @@ -163,6 +174,67 @@ public class NodeCLI extends YarnCLI { writer.flush(); } + /** + * Lists the nodes which are matching the given node states along with + * detailed node informations such as resource usage etc. + * + * @param nodeStates + * @throws YarnException + * @throws IOException + */ + private void listDetailedClusterNodes(Set nodeStates) + throws YarnException, IOException { + PrintWriter writer = new PrintWriter(new OutputStreamWriter(sysout, + Charset.forName("UTF-8"))); + List nodesReport = client.getNodeReports(nodeStates + .toArray(new NodeState[0])); + writer.println("Total Nodes:" + nodesReport.size()); + writer.printf(NODES_PATTERN, "Node-Id", "Node-State", "Node-Http-Address", + "Number-of-Running-Containers"); + for (NodeReport nodeReport : nodesReport) { + writer.printf(NODES_PATTERN, nodeReport.getNodeId(), + nodeReport.getNodeState(), nodeReport.getHttpAddress(), + nodeReport.getNumContainers()); + writer.println("Detailed Node Information :"); + writer.print("\tConfigured Resources : "); + writer.println(nodeReport.getCapability()); + writer.print("\tAllocated Resources : "); + if (nodeReport.getUsed() != null) { + writer.print(nodeReport.getUsed()); + } + writer.println(); + + writer.print("\tResource Utilization by Node : "); + if (nodeReport.getNodeUtilization() != null) { + writer.print("PMem:" + + nodeReport.getNodeUtilization().getPhysicalMemory() + + " MB, VMem:" + nodeReport.getNodeUtilization().getVirtualMemory() + + " MB, VCores:" + nodeReport.getNodeUtilization().getCPU()); + } + writer.println(); + + writer.print("\tResource Utilization by Containers : "); + if (nodeReport.getAggregatedContainersUtilization() != null) { + writer.print("PMem:" + + nodeReport.getAggregatedContainersUtilization() + .getPhysicalMemory() + + " MB, VMem:" + + nodeReport.getAggregatedContainersUtilization() + .getVirtualMemory() + " MB, VCores:" + + nodeReport.getAggregatedContainersUtilization().getCPU()); + } + writer.println(); + + writer.print("\tNode-Labels : "); + // Create a List for node labels since we need it get sorted + List nodeLabelsList = new ArrayList( + nodeReport.getNodeLabels()); + Collections.sort(nodeLabelsList); + writer.println(StringUtils.join(nodeLabelsList.iterator(), ',')); + } + writer.flush(); + } + /** * Prints the node report for node id. * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index f30bd40be0..1a77c7ca90 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -992,6 +992,35 @@ public class TestYarnCLI { Assert.assertEquals(nodesReportStr, sysOutStream.toString()); verify(sysOut, times(3)).write(any(byte[].class), anyInt(), anyInt()); + sysOutStream.reset(); + result = cli.run(new String[] { "-list", "-showDetails" }); + assertEquals(0, result); + baos = new ByteArrayOutputStream(); + pw = new PrintWriter(baos); + pw.println("Total Nodes:2"); + pw.print(" Node-Id\t Node-State\tNode-Http-Address\t"); + pw.println("Number-of-Running-Containers"); + pw.print(" host0:0\t RUNNING\t host1:8888\t"); + pw.println(" 0"); + pw.println("Detailed Node Information :"); + pw.println("\tConfigured Resources : "); + pw.println("\tAllocated Resources : "); + pw.println("\tResource Utilization by Node : PMem:2048 MB, VMem:4096 MB, VCores:8.0"); + pw.println("\tResource Utilization by Containers : PMem:1024 MB, VMem:2048 MB, VCores:4.0"); + pw.println("\tNode-Labels : "); + pw.print(" host1:0\t RUNNING\t host1:8888\t"); + pw.println(" 0"); + pw.println("Detailed Node Information :"); + pw.println("\tConfigured Resources : "); + pw.println("\tAllocated Resources : "); + pw.println("\tResource Utilization by Node : PMem:2048 MB, VMem:4096 MB, VCores:8.0"); + pw.println("\tResource Utilization by Containers : PMem:1024 MB, VMem:2048 MB, VCores:4.0"); + pw.println("\tNode-Labels : "); + pw.close(); + nodesReportStr = baos.toString("UTF-8"); + Assert.assertEquals(nodesReportStr, sysOutStream.toString()); + verify(sysOut, times(4)).write(any(byte[].class), anyInt(), anyInt()); + sysOutStream.reset(); nodeStates.clear(); nodeStates.add(NodeState.UNHEALTHY); @@ -1011,7 +1040,7 @@ public class TestYarnCLI { pw.close(); nodesReportStr = baos.toString("UTF-8"); Assert.assertEquals(nodesReportStr, sysOutStream.toString()); - verify(sysOut, times(4)).write(any(byte[].class), anyInt(), anyInt()); + verify(sysOut, times(5)).write(any(byte[].class), anyInt(), anyInt()); sysOutStream.reset(); nodeStates.clear(); @@ -1032,7 +1061,7 @@ public class TestYarnCLI { pw.close(); nodesReportStr = baos.toString("UTF-8"); Assert.assertEquals(nodesReportStr, sysOutStream.toString()); - verify(sysOut, times(5)).write(any(byte[].class), anyInt(), anyInt()); + verify(sysOut, times(6)).write(any(byte[].class), anyInt(), anyInt()); sysOutStream.reset(); nodeStates.clear(); @@ -1053,7 +1082,7 @@ public class TestYarnCLI { pw.close(); nodesReportStr = baos.toString("UTF-8"); Assert.assertEquals(nodesReportStr, sysOutStream.toString()); - verify(sysOut, times(6)).write(any(byte[].class), anyInt(), anyInt()); + verify(sysOut, times(7)).write(any(byte[].class), anyInt(), anyInt()); sysOutStream.reset(); nodeStates.clear(); @@ -1074,7 +1103,7 @@ public class TestYarnCLI { pw.close(); nodesReportStr = baos.toString("UTF-8"); Assert.assertEquals(nodesReportStr, sysOutStream.toString()); - verify(sysOut, times(7)).write(any(byte[].class), anyInt(), anyInt()); + verify(sysOut, times(8)).write(any(byte[].class), anyInt(), anyInt()); sysOutStream.reset(); nodeStates.clear(); @@ -1107,7 +1136,7 @@ public class TestYarnCLI { pw.close(); nodesReportStr = baos.toString("UTF-8"); Assert.assertEquals(nodesReportStr, sysOutStream.toString()); - verify(sysOut, times(8)).write(any(byte[].class), anyInt(), anyInt()); + verify(sysOut, times(9)).write(any(byte[].class), anyInt(), anyInt()); sysOutStream.reset(); nodeStates.clear(); @@ -1142,7 +1171,7 @@ public class TestYarnCLI { pw.close(); nodesReportStr = baos.toString("UTF-8"); Assert.assertEquals(nodesReportStr, sysOutStream.toString()); - verify(sysOut, times(9)).write(any(byte[].class), anyInt(), anyInt()); + verify(sysOut, times(10)).write(any(byte[].class), anyInt(), anyInt()); } private List getNodeReports( @@ -1727,7 +1756,9 @@ public class TestYarnCLI { pw.println(" -help Displays help for all commands."); pw.println(" -list List all running nodes. Supports optional use of"); pw.println(" -states to filter nodes based on node state, all -all"); - pw.println(" to list all nodes."); + pw.println(" to list all nodes, -showDetails to display more"); + pw.println(" details about each node."); + pw.println(" -showDetails Works with -list to show more details about each node."); pw.println(" -states Works with -list to filter nodes based on input"); pw.println(" comma-separated list of node states."); pw.println(" -status Prints the status report of the node.");