YARN-2786. Created a yarn cluster CLI and seeded with one command for listing node-labels collection. Contributed by Wangda Tan.
This commit is contained in:
Vinod Kumar Vavilapalli
parent
28b85a2116
commit
138c9cadee
@ -97,6 +97,9 @@ Release 2.7.0 - UNRELEASED
|
||||
YARN-3249. Add a 'kill application' button to Resource Manager's Web UI.
|
||||
(Ryu Kobayashi via ozawa)
|
||||
|
||||
YARN-2786. Created a yarn cluster CLI and seeded with one command for listing
|
||||
node-labels collection. (Wangda Tan via vinodkv)
|
||||
|
||||
IMPROVEMENTS
|
||||
|
||||
YARN-3005. [JDK7] Use switch statement for String instead of if-else
|
||||
|
||||
@ -25,6 +25,7 @@ function hadoop_usage
|
||||
echo " applicationattempt prints applicationattempt(s) report"
|
||||
echo " classpath prints the class path needed to get the"
|
||||
echo " Hadoop jar and the required libraries"
|
||||
echo " cluster prints cluster information"
|
||||
echo " container prints container(s) report"
|
||||
echo " daemonlog get/set the log level for each daemon"
|
||||
echo " jar <jar> run a jar file"
|
||||
@ -83,6 +84,11 @@ case "${COMMAND}" in
|
||||
classpath)
|
||||
hadoop_do_classpath_subcommand "$@"
|
||||
;;
|
||||
cluster)
|
||||
CLASS=org.apache.hadoop.yarn.client.cli.ClusterCLI
|
||||
hadoop_debug "Append YARN_CLIENT_OPTS onto YARN_OPTS"
|
||||
YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
|
||||
;;
|
||||
daemonlog)
|
||||
CLASS=org.apache.hadoop.log.LogLevel
|
||||
hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
|
||||
|
||||
@ -192,6 +192,11 @@ goto :eof
|
||||
set yarn-command-arguments=%yarn-command% %yarn-command-arguments%
|
||||
goto :eof
|
||||
|
||||
:cluster
|
||||
set CLASS=org.apache.hadoop.yarn.client.cli.ClusterCLI
|
||||
set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
|
||||
goto :eof
|
||||
|
||||
:container
|
||||
set CLASS=org.apache.hadoop.yarn.client.cli.ApplicationCLI
|
||||
set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
|
||||
@ -312,6 +317,7 @@ goto :eof
|
||||
@echo jar ^<jar^> run a jar file
|
||||
@echo application prints application(s) report/kill application
|
||||
@echo applicationattempt prints applicationattempt(s) report
|
||||
@echo cluster prints cluster information
|
||||
@echo container prints container(s) report
|
||||
@echo node prints node report(s)
|
||||
@echo queue prints queue information
|
||||
|
||||
@ -0,0 +1,157 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.client.cli;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.io.PrintWriter;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.commons.cli.CommandLine;
|
||||
import org.apache.commons.cli.GnuParser;
|
||||
import org.apache.commons.cli.HelpFormatter;
|
||||
import org.apache.commons.cli.MissingArgumentException;
|
||||
import org.apache.commons.cli.Options;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.util.ToolRunner;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
|
||||
/**
|
||||
* Cluster CLI used to get over all information of the cluster
|
||||
*/
|
||||
@Private
|
||||
public class ClusterCLI extends YarnCLI {
|
||||
private static final String TITLE = "yarn cluster";
|
||||
public static final String LIST_LABELS_CMD = "list-node-labels";
|
||||
public static final String DIRECTLY_ACCESS_NODE_LABEL_STORE =
|
||||
"directly-access-node-label-store";
|
||||
public static final String CMD = "cluster";
|
||||
private boolean accessLocal = false;
|
||||
static CommonNodeLabelsManager localNodeLabelsManager = null;
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
ClusterCLI cli = new ClusterCLI();
|
||||
cli.setSysOutPrintStream(System.out);
|
||||
cli.setSysErrPrintStream(System.err);
|
||||
int res = ToolRunner.run(cli, args);
|
||||
cli.stop();
|
||||
System.exit(res);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int run(String[] args) throws Exception {
|
||||
Options opts = new Options();
|
||||
|
||||
opts.addOption("lnl", LIST_LABELS_CMD, false,
|
||||
"List cluster node-label collection");
|
||||
opts.addOption("h", HELP_CMD, false, "Displays help for all commands.");
|
||||
opts.addOption("dnl", DIRECTLY_ACCESS_NODE_LABEL_STORE, false,
|
||||
"Directly access node label store, "
|
||||
+ "with this option, all node label related operations"
|
||||
+ " will NOT connect RM. Instead, they will"
|
||||
+ " access/modify stored node labels directly."
|
||||
+ " By default, it is false (access via RM)."
|
||||
+ " AND PLEASE NOTE: if you configured "
|
||||
+ YarnConfiguration.FS_NODE_LABELS_STORE_ROOT_DIR
|
||||
+ " to a local directory"
|
||||
+ " (instead of NFS or HDFS), this option will only work"
|
||||
+ " when the command run on the machine where RM is running."
|
||||
+ " Also, this option is UNSTABLE, could be removed in future"
|
||||
+ " releases.");
|
||||
|
||||
int exitCode = -1;
|
||||
CommandLine parsedCli = null;
|
||||
try {
|
||||
parsedCli = new GnuParser().parse(opts, args);
|
||||
} catch (MissingArgumentException ex) {
|
||||
sysout.println("Missing argument for options");
|
||||
printUsage(opts);
|
||||
return exitCode;
|
||||
}
|
||||
|
||||
if (parsedCli.hasOption(DIRECTLY_ACCESS_NODE_LABEL_STORE)) {
|
||||
accessLocal = true;
|
||||
}
|
||||
|
||||
if (parsedCli.hasOption(LIST_LABELS_CMD)) {
|
||||
printClusterNodeLabels();
|
||||
} else if (parsedCli.hasOption(HELP_CMD)) {
|
||||
printUsage(opts);
|
||||
return 0;
|
||||
} else {
|
||||
syserr.println("Invalid Command Usage : ");
|
||||
printUsage(opts);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
private List<String> sortStrSet(Set<String> labels) {
|
||||
List<String> list = new ArrayList<String>();
|
||||
list.addAll(labels);
|
||||
Collections.sort(list);
|
||||
return list;
|
||||
}
|
||||
|
||||
void printClusterNodeLabels() throws YarnException, IOException {
|
||||
Set<String> nodeLabels = null;
|
||||
if (accessLocal) {
|
||||
nodeLabels =
|
||||
getNodeLabelManagerInstance(getConf()).getClusterNodeLabels();
|
||||
} else {
|
||||
nodeLabels = client.getClusterNodeLabels();
|
||||
}
|
||||
sysout.println(String.format("Node Labels: %s",
|
||||
StringUtils.join(sortStrSet(nodeLabels).iterator(), ",")));
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
static synchronized CommonNodeLabelsManager
|
||||
getNodeLabelManagerInstance(Configuration conf) {
|
||||
if (localNodeLabelsManager == null) {
|
||||
localNodeLabelsManager = new CommonNodeLabelsManager();
|
||||
localNodeLabelsManager.init(conf);
|
||||
localNodeLabelsManager.start();
|
||||
}
|
||||
return localNodeLabelsManager;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
void printUsage(Options opts) throws UnsupportedEncodingException {
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
PrintWriter pw =
|
||||
new PrintWriter(new OutputStreamWriter(baos, Charset.forName("UTF-8")));
|
||||
new HelpFormatter().printHelp(pw, HelpFormatter.DEFAULT_WIDTH, TITLE, null,
|
||||
opts, HelpFormatter.DEFAULT_LEFT_PAD, HelpFormatter.DEFAULT_DESC_PAD,
|
||||
null);
|
||||
pw.close();
|
||||
sysout.println(baos.toString("UTF-8"));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,158 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.client.cli;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.spy;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.PrintStream;
|
||||
import java.io.PrintWriter;
|
||||
import java.util.HashSet;
|
||||
|
||||
import org.apache.hadoop.yarn.client.api.YarnClient;
|
||||
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
|
||||
public class TestClusterCLI {
|
||||
ByteArrayOutputStream sysOutStream;
|
||||
private PrintStream sysOut;
|
||||
ByteArrayOutputStream sysErrStream;
|
||||
private PrintStream sysErr;
|
||||
|
||||
@Before
|
||||
public void setup() {
|
||||
sysOutStream = new ByteArrayOutputStream();
|
||||
sysOut = spy(new PrintStream(sysOutStream));
|
||||
sysErrStream = new ByteArrayOutputStream();
|
||||
sysErr = spy(new PrintStream(sysErrStream));
|
||||
System.setOut(sysOut);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetClusterNodeLabels() throws Exception {
|
||||
YarnClient client = mock(YarnClient.class);
|
||||
when(client.getClusterNodeLabels()).thenReturn(
|
||||
ImmutableSet.of("label1", "label2"));
|
||||
ClusterCLI cli = new ClusterCLI();
|
||||
cli.setClient(client);
|
||||
cli.setSysOutPrintStream(sysOut);
|
||||
cli.setSysErrPrintStream(sysErr);
|
||||
|
||||
int rc =
|
||||
cli.run(new String[] { ClusterCLI.CMD, "-" + ClusterCLI.LIST_LABELS_CMD });
|
||||
assertEquals(0, rc);
|
||||
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
PrintWriter pw = new PrintWriter(baos);
|
||||
pw.print("Node Labels: label1,label2");
|
||||
pw.close();
|
||||
verify(sysOut).println(baos.toString("UTF-8"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetClusterNodeLabelsWithLocalAccess() throws Exception {
|
||||
YarnClient client = mock(YarnClient.class);
|
||||
when(client.getClusterNodeLabels()).thenReturn(
|
||||
ImmutableSet.of("remote1", "remote2"));
|
||||
ClusterCLI cli = new ClusterCLI();
|
||||
cli.setClient(client);
|
||||
cli.setSysOutPrintStream(sysOut);
|
||||
cli.setSysErrPrintStream(sysErr);
|
||||
ClusterCLI.localNodeLabelsManager = mock(CommonNodeLabelsManager.class);
|
||||
when(ClusterCLI.localNodeLabelsManager.getClusterNodeLabels())
|
||||
.thenReturn(ImmutableSet.of("local1", "local2"));
|
||||
|
||||
int rc =
|
||||
cli.run(new String[] { ClusterCLI.CMD,
|
||||
"-" + ClusterCLI.LIST_LABELS_CMD,
|
||||
"-" + ClusterCLI.DIRECTLY_ACCESS_NODE_LABEL_STORE });
|
||||
assertEquals(0, rc);
|
||||
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
PrintWriter pw = new PrintWriter(baos);
|
||||
// it should return local* instead of remote*
|
||||
pw.print("Node Labels: local1,local2");
|
||||
pw.close();
|
||||
verify(sysOut).println(baos.toString("UTF-8"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetEmptyClusterNodeLabels() throws Exception {
|
||||
YarnClient client = mock(YarnClient.class);
|
||||
when(client.getClusterNodeLabels()).thenReturn(new HashSet<String>());
|
||||
ClusterCLI cli = new ClusterCLI();
|
||||
cli.setClient(client);
|
||||
cli.setSysOutPrintStream(sysOut);
|
||||
cli.setSysErrPrintStream(sysErr);
|
||||
|
||||
int rc =
|
||||
cli.run(new String[] { ClusterCLI.CMD, "-" + ClusterCLI.LIST_LABELS_CMD });
|
||||
assertEquals(0, rc);
|
||||
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
PrintWriter pw = new PrintWriter(baos);
|
||||
pw.print("Node Labels: ");
|
||||
pw.close();
|
||||
verify(sysOut).println(baos.toString("UTF-8"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testHelp() throws Exception {
|
||||
ClusterCLI cli = new ClusterCLI();
|
||||
cli.setSysOutPrintStream(sysOut);
|
||||
cli.setSysErrPrintStream(sysErr);
|
||||
|
||||
int rc =
|
||||
cli.run(new String[] { "cluster", "--help" });
|
||||
assertEquals(0, rc);
|
||||
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
PrintWriter pw = new PrintWriter(baos);
|
||||
pw.println("usage: yarn cluster");
|
||||
pw.println(" -dnl,--directly-access-node-label-store Directly access node label");
|
||||
pw.println(" store, with this option, all");
|
||||
pw.println(" node label related operations");
|
||||
pw.println(" will NOT connect RM. Instead,");
|
||||
pw.println(" they will access/modify stored");
|
||||
pw.println(" node labels directly. By");
|
||||
pw.println(" default, it is false (access");
|
||||
pw.println(" via RM). AND PLEASE NOTE: if");
|
||||
pw.println(" you configured");
|
||||
pw.println(" yarn.node-labels.fs-store.root-");
|
||||
pw.println(" dir to a local directory");
|
||||
pw.println(" (instead of NFS or HDFS), this");
|
||||
pw.println(" option will only work when the");
|
||||
pw.println(" command run on the machine");
|
||||
pw.println(" where RM is running. Also, this");
|
||||
pw.println(" option is UNSTABLE, could be");
|
||||
pw.println(" removed in future releases.");
|
||||
pw.println(" -h,--help Displays help for all commands.");
|
||||
pw.println(" -lnl,--list-node-labels List cluster node-label");
|
||||
pw.println(" collection");
|
||||
pw.close();
|
||||
verify(sysOut).println(baos.toString("UTF-8"));
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user