YARN-2786. Created a yarn cluster CLI and seeded with one command for listing node-labels collection. Contributed by Wangda Tan.

hadoop-yarn-project/CHANGES.txt

@@ -97,6 +97,9 @@ Release 2.7.0 - UNRELEASED
     YARN-3249. Add a 'kill application' button to Resource Manager's Web UI.
     (Ryu Kobayashi via ozawa)
 
+    YARN-2786. Created a yarn cluster CLI and seeded with one command for listing
+    node-labels collection. (Wangda Tan via vinodkv)
+
   IMPROVEMENTS
 
     YARN-3005. [JDK7] Use switch statement for String instead of if-else

hadoop-yarn-project/hadoop-yarn/bin/yarn

@@ -25,6 +25,7 @@ function hadoop_usage
   echo "  applicationattempt                    prints applicationattempt(s) report"
   echo "  classpath                             prints the class path needed to get the"
   echo "                                        Hadoop jar and the required libraries"
+  echo "  cluster                               prints cluster information"
   echo "  container                             prints container(s) report"
   echo "  daemonlog                             get/set the log level for each daemon"
   echo "  jar <jar>                             run a jar file"
@@ -83,6 +84,11 @@ case "${COMMAND}" in
   classpath)
     hadoop_do_classpath_subcommand "$@"
   ;;
+  cluster)
+    CLASS=org.apache.hadoop.yarn.client.cli.ClusterCLI
+    hadoop_debug "Append YARN_CLIENT_OPTS onto YARN_OPTS"
+    YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
+  ;;
   daemonlog)
     CLASS=org.apache.hadoop.log.LogLevel
     hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"

hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd

@@ -192,6 +192,11 @@ goto :eof
   set yarn-command-arguments=%yarn-command% %yarn-command-arguments%
   goto :eof
 
+:cluster
+  set CLASS=org.apache.hadoop.yarn.client.cli.ClusterCLI
+  set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
+  goto :eof
+
 :container
   set CLASS=org.apache.hadoop.yarn.client.cli.ApplicationCLI
   set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
@@ -312,6 +317,7 @@ goto :eof
   @echo   jar ^<jar^>          run a jar file
   @echo   application          prints application(s) report/kill application
   @echo   applicationattempt   prints applicationattempt(s) report
+  @echo   cluster              prints cluster information
   @echo   container            prints container(s) report
   @echo   node                 prints node report(s)
   @echo   queue                prints queue information

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ClusterCLI.java

@@ -0,0 +1,157 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.client.cli;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.io.UnsupportedEncodingException;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.MissingArgumentException;
+import org.apache.commons.cli.Options;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * Cluster CLI used to get over all information of the cluster
+ */
+@Private
+public class ClusterCLI extends YarnCLI {
+  private static final String TITLE = "yarn cluster";
+  public static final String LIST_LABELS_CMD = "list-node-labels";
+  public static final String DIRECTLY_ACCESS_NODE_LABEL_STORE =
+      "directly-access-node-label-store";
+  public static final String CMD = "cluster";
+  private boolean accessLocal = false;
+  static CommonNodeLabelsManager localNodeLabelsManager = null;
+
+  public static void main(String[] args) throws Exception {
+    ClusterCLI cli = new ClusterCLI();
+    cli.setSysOutPrintStream(System.out);
+    cli.setSysErrPrintStream(System.err);
+    int res = ToolRunner.run(cli, args);
+    cli.stop();
+    System.exit(res);
+  }
+
+  @Override
+  public int run(String[] args) throws Exception {    
+    Options opts = new Options();
+
+    opts.addOption("lnl", LIST_LABELS_CMD, false,
+        "List cluster node-label collection");
+    opts.addOption("h", HELP_CMD, false, "Displays help for all commands.");
+    opts.addOption("dnl", DIRECTLY_ACCESS_NODE_LABEL_STORE, false,
+        "Directly access node label store, "
+            + "with this option, all node label related operations"
+            + " will NOT connect RM. Instead, they will"
+            + " access/modify stored node labels directly."
+            + " By default, it is false (access via RM)."
+            + " AND PLEASE NOTE: if you configured "
+            + YarnConfiguration.FS_NODE_LABELS_STORE_ROOT_DIR
+            + " to a local directory"
+            + " (instead of NFS or HDFS), this option will only work"
+            + " when the command run on the machine where RM is running."
+            + " Also, this option is UNSTABLE, could be removed in future"
+            + " releases.");
+
+    int exitCode = -1;
+    CommandLine parsedCli = null;
+    try {
+      parsedCli = new GnuParser().parse(opts, args);
+    } catch (MissingArgumentException ex) {
+      sysout.println("Missing argument for options");
+      printUsage(opts);
+      return exitCode;
+    }
+
+    if (parsedCli.hasOption(DIRECTLY_ACCESS_NODE_LABEL_STORE)) {
+      accessLocal = true;
+    }
+
+    if (parsedCli.hasOption(LIST_LABELS_CMD)) {
+      printClusterNodeLabels();
+    } else if (parsedCli.hasOption(HELP_CMD)) {
+      printUsage(opts);
+      return 0;
+    } else {
+      syserr.println("Invalid Command Usage : ");
+      printUsage(opts);
+    }
+    return 0;
+  }
+
+  private List<String> sortStrSet(Set<String> labels) {
+    List<String> list = new ArrayList<String>();
+    list.addAll(labels);
+    Collections.sort(list);
+    return list;
+  }
+
+  void printClusterNodeLabels() throws YarnException, IOException {
+    Set<String> nodeLabels = null;
+    if (accessLocal) {
+      nodeLabels =
+          getNodeLabelManagerInstance(getConf()).getClusterNodeLabels();
+    } else {
+      nodeLabels = client.getClusterNodeLabels();
+    }
+    sysout.println(String.format("Node Labels: %s",
+        StringUtils.join(sortStrSet(nodeLabels).iterator(), ",")));
+  }
+
+  @VisibleForTesting
+  static synchronized CommonNodeLabelsManager
+      getNodeLabelManagerInstance(Configuration conf) {
+    if (localNodeLabelsManager == null) {
+      localNodeLabelsManager = new CommonNodeLabelsManager();
+      localNodeLabelsManager.init(conf);
+      localNodeLabelsManager.start();
+    }
+    return localNodeLabelsManager;
+  }
+
+  @VisibleForTesting
+  void printUsage(Options opts) throws UnsupportedEncodingException {
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    PrintWriter pw =
+        new PrintWriter(new OutputStreamWriter(baos, Charset.forName("UTF-8")));
+    new HelpFormatter().printHelp(pw, HelpFormatter.DEFAULT_WIDTH, TITLE, null,
+        opts, HelpFormatter.DEFAULT_LEFT_PAD, HelpFormatter.DEFAULT_DESC_PAD,
+        null);
+    pw.close();
+    sysout.println(baos.toString("UTF-8"));
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestClusterCLI.java

@@ -0,0 +1,158 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.client.cli;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.io.PrintWriter;
+import java.util.HashSet;
+
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.ImmutableSet;
+
+public class TestClusterCLI {
+  ByteArrayOutputStream sysOutStream;
+  private PrintStream sysOut;
+  ByteArrayOutputStream sysErrStream;
+  private PrintStream sysErr;
+
+  @Before
+  public void setup() {
+    sysOutStream = new ByteArrayOutputStream();
+    sysOut = spy(new PrintStream(sysOutStream));
+    sysErrStream = new ByteArrayOutputStream();
+    sysErr = spy(new PrintStream(sysErrStream));
+    System.setOut(sysOut);
+  }
+  
+  @Test
+  public void testGetClusterNodeLabels() throws Exception {
+    YarnClient client = mock(YarnClient.class);
+    when(client.getClusterNodeLabels()).thenReturn(
+        ImmutableSet.of("label1", "label2"));
+    ClusterCLI cli = new ClusterCLI();
+    cli.setClient(client);
+    cli.setSysOutPrintStream(sysOut);
+    cli.setSysErrPrintStream(sysErr);
+    
+    int rc =
+        cli.run(new String[] { ClusterCLI.CMD, "-" + ClusterCLI.LIST_LABELS_CMD });
+    assertEquals(0, rc);
+    
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    PrintWriter pw = new PrintWriter(baos);
+    pw.print("Node Labels: label1,label2");
+    pw.close();
+    verify(sysOut).println(baos.toString("UTF-8"));
+  }
+  
+  @Test
+  public void testGetClusterNodeLabelsWithLocalAccess() throws Exception {
+    YarnClient client = mock(YarnClient.class);
+    when(client.getClusterNodeLabels()).thenReturn(
+        ImmutableSet.of("remote1", "remote2"));
+    ClusterCLI cli = new ClusterCLI();
+    cli.setClient(client);
+    cli.setSysOutPrintStream(sysOut);
+    cli.setSysErrPrintStream(sysErr);
+    ClusterCLI.localNodeLabelsManager = mock(CommonNodeLabelsManager.class);
+    when(ClusterCLI.localNodeLabelsManager.getClusterNodeLabels())
+        .thenReturn(ImmutableSet.of("local1", "local2"));
+
+    int rc =
+        cli.run(new String[] { ClusterCLI.CMD,
+            "-" + ClusterCLI.LIST_LABELS_CMD,
+            "-" + ClusterCLI.DIRECTLY_ACCESS_NODE_LABEL_STORE });
+    assertEquals(0, rc);
+
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    PrintWriter pw = new PrintWriter(baos);
+    // it should return local* instead of remote*
+    pw.print("Node Labels: local1,local2");
+    pw.close();
+    verify(sysOut).println(baos.toString("UTF-8"));
+  }
+  
+  @Test
+  public void testGetEmptyClusterNodeLabels() throws Exception {
+    YarnClient client = mock(YarnClient.class);
+    when(client.getClusterNodeLabels()).thenReturn(new HashSet<String>());
+    ClusterCLI cli = new ClusterCLI();
+    cli.setClient(client);
+    cli.setSysOutPrintStream(sysOut);
+    cli.setSysErrPrintStream(sysErr);
+
+    int rc =
+        cli.run(new String[] { ClusterCLI.CMD, "-" + ClusterCLI.LIST_LABELS_CMD });
+    assertEquals(0, rc);
+
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    PrintWriter pw = new PrintWriter(baos);
+    pw.print("Node Labels: ");
+    pw.close();
+    verify(sysOut).println(baos.toString("UTF-8"));
+  }
+  
+  @Test
+  public void testHelp() throws Exception {
+    ClusterCLI cli = new ClusterCLI();
+    cli.setSysOutPrintStream(sysOut);
+    cli.setSysErrPrintStream(sysErr);
+
+    int rc =
+        cli.run(new String[] { "cluster", "--help" });
+    assertEquals(0, rc);
+    
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    PrintWriter pw = new PrintWriter(baos);
+    pw.println("usage: yarn cluster");
+    pw.println(" -dnl,--directly-access-node-label-store   Directly access node label");
+    pw.println("                                           store, with this option, all");
+    pw.println("                                           node label related operations");
+    pw.println("                                           will NOT connect RM. Instead,");
+    pw.println("                                           they will access/modify stored");
+    pw.println("                                           node labels directly. By");
+    pw.println("                                           default, it is false (access");
+    pw.println("                                           via RM). AND PLEASE NOTE: if");
+    pw.println("                                           you configured");
+    pw.println("                                           yarn.node-labels.fs-store.root-");
+    pw.println("                                           dir to a local directory");
+    pw.println("                                           (instead of NFS or HDFS), this");
+    pw.println("                                           option will only work when the");
+    pw.println("                                           command run on the machine");
+    pw.println("                                           where RM is running. Also, this");
+    pw.println("                                           option is UNSTABLE, could be");
+    pw.println("                                           removed in future releases.");
+    pw.println(" -h,--help                                 Displays help for all commands.");
+    pw.println(" -lnl,--list-node-labels                   List cluster node-label");
+    pw.println("                                           collection");
+    pw.close();
+    verify(sysOut).println(baos.toString("UTF-8"));
+  }
+}