big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-2647. Added a queue CLI for getting queue information. Contributed by Sunil Govind.

Browse Source
This commit is contained in:
Vinod Kumar Vavilapalli 2014-11-06 20:48:31 -08:00
parent ba0a42cf67
commit 61effcbebc
5 changed files with 224 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -192,6 +192,9 @@ Release 2.6.0 - UNRELEASED
YARN-2198. Remove the need to run NodeManager as privileged account for
Windows Secure Container Executor. (Remus Rusanu via jianhe)
YARN-2647. Added a queue CLI for getting queue information. (Sunil Govind via
vinodkv)
IMPROVEMENTS
YARN-2197. Add a link to YARN CHANGES.txt in the left side of doc

6
hadoop-yarn-project/hadoop-yarn/bin/yarn
View File

@ -30,6 +30,7 @@ function hadoop_usage
echo " node prints node report(s)"
echo " nodemanager run a nodemanager on each slave"
echo " proxyserver run the web app proxy server"
echo " queue prints queue information"
echo " resourcemanager run the ResourceManager"
echo " resourcemanager -format-state-store deletes the RMStateStore"
echo " rmadmin admin tools"
@ -127,6 +128,11 @@ case "${COMMAND}" in
JAVA_HEAP_MAX="-Xmx${YARN_PROXYSERVER_HEAPSIZE}m"
fi
;;
queue)
CLASS=org.apache.hadoop.yarn.client.cli.QueueCLI
hadoop_debug "Append YARN_CLIENT_OPTS onto YARN_OPTS"
YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
;;
resourcemanager)
daemon="true"
CLASS='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager'

6
hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
View File

@ -196,6 +196,11 @@ goto :eof
set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
goto :eof
:queue
set CLASS=org.apache.hadoop.yarn.client.cli.QueueCLI
set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
goto :eof
:resourcemanager
set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\rm-config\log4j.properties
set CLASS=org.apache.hadoop.yarn.server.resourcemanager.ResourceManager
@ -298,6 +303,7 @@ goto :eof
@echo applicationattempt prints applicationattempt(s) report
@echo container prints container(s) report
@echo node prints node report(s)
@echo queue prints queue information
@echo logs dump container logs
@echo classpath prints the class path needed to get the
@echo Hadoop jar and the required libraries

148
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java Normal file
View File

@ -0,0 +1,148 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.client.cli;
import java.io.IOException;
import java.io.PrintWriter;
import java.text.DecimalFormat;
import java.util.Set;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.MissingArgumentException;
import org.apache.commons.cli.Options;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.exceptions.YarnException;
import com.google.common.annotations.VisibleForTesting;
@Private
@Unstable
public class QueueCLI extends YarnCLI {
public static final String QUEUE = "queue";
public static void main(String[] args) throws Exception {
QueueCLI cli = new QueueCLI();
cli.setSysOutPrintStream(System.out);
cli.setSysErrPrintStream(System.err);
int res = ToolRunner.run(cli, args);
cli.stop();
System.exit(res);
}
@Override
public int run(String[] args) throws Exception {
Options opts = new Options();
int exitCode = -1;
if (args.length > 0) {
opts.addOption(STATUS_CMD, true,
"List queue information about given queue.");
opts.addOption(HELP_CMD, false, "Displays help for all commands.");
opts.getOption(STATUS_CMD).setArgName("Queue Name");
} else {
syserr.println("Invalid Command usage. Command must start with 'queue'");
return exitCode;
}
CommandLine cliParser = null;
try {
cliParser = new GnuParser().parse(opts, args);
} catch (MissingArgumentException ex) {
sysout.println("Missing argument for options");
printUsage(opts);
return exitCode;
}
if (cliParser.hasOption(STATUS_CMD)) {
if (args.length != 3) {
printUsage(opts);
return exitCode;
}
listQueue(cliParser.getOptionValue(STATUS_CMD));
} else if (cliParser.hasOption(HELP_CMD)) {
printUsage(opts);
return 0;
} else {
syserr.println("Invalid Command Usage : ");
printUsage(opts);
}
return 0;
}
/**
* It prints the usage of the command
*
* @param opts
*/
@VisibleForTesting
void printUsage(Options opts) {
new HelpFormatter().printHelp(QUEUE, opts);
}
/**
* Lists the Queue Information matching the given queue name
*
* @param queueName
* @throws YarnException
* @throws IOException
*/
private void listQueue(String queueName) throws YarnException, IOException {
PrintWriter writer = new PrintWriter(sysout);
QueueInfo queueInfo = client.getQueueInfo(queueName);
writer.println("Queue Information : ");
printQueueInfo(writer, queueInfo);
writer.flush();
}
private void printQueueInfo(PrintWriter writer, QueueInfo queueInfo) {
writer.print("Queue Name : ");
writer.println(queueInfo.getQueueName());
writer.print("\tState : ");
writer.println(queueInfo.getQueueState());
DecimalFormat df = new DecimalFormat("#.0");
writer.print("\tCapacity : ");
writer.println(df.format(queueInfo.getCapacity() * 100) + "%");
writer.print("\tCurrent Capacity : ");
writer.println(df.format(queueInfo.getCurrentCapacity() * 100) + "%");
writer.print("\tMaximum Capacity : ");
writer.println(df.format(queueInfo.getMaximumCapacity() * 100) + "%");
writer.print("\tDefault Node Label expression : ");
if (null != queueInfo.getDefaultNodeLabelExpression()) {
writer.println(queueInfo.getDefaultNodeLabelExpression());
} else {
writer.println();
}
Set<String> nodeLabels = queueInfo.getAccessibleNodeLabels();
StringBuilder labelList = new StringBuilder();
writer.print("\tAccessible Node Labels : ");
for (String nodeLabel : nodeLabels) {
if (labelList.length() > 0) {
labelList.append(',');
}
labelList.append(nodeLabel);
}
writer.println(labelList.toString());
}
}

61
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
View File

@ -58,6 +58,8 @@
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
@ -1235,6 +1237,57 @@ public void testMissingArguments() throws Exception {
Assert.assertEquals(String.format("Missing argument for options%n%1s",
createNodeCLIHelpMessage()), sysOutStream.toString());
}
@Test
public void testGetQueueInfo() throws Exception {
QueueCLI cli = createAndGetQueueCLI();
Set<String> nodeLabels = new HashSet<String>();
nodeLabels.add("GPU");
nodeLabels.add("JDK_7");
QueueInfo queueInfo = QueueInfo.newInstance("queueA", 0.4f, 0.8f, 0.5f,
null, null, QueueState.RUNNING, nodeLabels, "GPU");
when(client.getQueueInfo(any(String.class))).thenReturn(queueInfo);
int result = cli.run(new String[] { "queue", "-status", "queueA" });
assertEquals(0, result);
verify(client).getQueueInfo("queueA");
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw = new PrintWriter(baos);
pw.println("Queue Information : ");
pw.println("Queue Name : " + "queueA");
pw.println("\tState : " + "RUNNING");
pw.println("\tCapacity : " + "40.0%");
pw.println("\tCurrent Capacity : " + "50.0%");
pw.println("\tMaximum Capacity : " + "80.0%");
pw.println("\tDefault Node Label expression : " + "GPU");
pw.println("\tAccessible Node Labels : " + "JDK_7,GPU");
pw.close();
String queueInfoStr = baos.toString("UTF-8");
Assert.assertEquals(queueInfoStr, sysOutStream.toString());
}
@Test
public void testGetQueueInfoWithEmptyNodeLabel() throws Exception {
QueueCLI cli = createAndGetQueueCLI();
QueueInfo queueInfo = QueueInfo.newInstance("queueA", 0.4f, 0.8f, 0.5f,
null, null, QueueState.RUNNING, null, null);
when(client.getQueueInfo(any(String.class))).thenReturn(queueInfo);
int result = cli.run(new String[] { "queue", "-status", "queueA" });
assertEquals(0, result);
verify(client).getQueueInfo("queueA");
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw = new PrintWriter(baos);
pw.println("Queue Information : ");
pw.println("Queue Name : " + "queueA");
pw.println("\tState : " + "RUNNING");
pw.println("\tCapacity : " + "40.0%");
pw.println("\tCurrent Capacity : " + "50.0%");
pw.println("\tMaximum Capacity : " + "80.0%");
pw.println("\tDefault Node Label expression : ");
pw.println("\tAccessible Node Labels : ");
pw.close();
String queueInfoStr = baos.toString("UTF-8");
Assert.assertEquals(queueInfoStr, sysOutStream.toString());
}
private void verifyUsageInfo(YarnCLI cli) throws Exception {
cli.setSysErrPrintStream(sysErr);
@ -1272,6 +1325,14 @@ private ApplicationCLI createAndGetAppCLI() {
cli.setSysOutPrintStream(sysOut);
return cli;
}
private QueueCLI createAndGetQueueCLI() {
QueueCLI cli = new QueueCLI();
cli.setClient(client);
cli.setSysOutPrintStream(sysOut);
cli.setSysErrPrintStream(sysErr);
return cli;
}
private String createApplicationCLIHelpMessage() throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();