From eb08543c7a5d1fd97a1915dbc5a11a2ba2066ba1 Mon Sep 17 00:00:00 2001 From: Naganarasimha Date: Thu, 28 Jun 2018 08:13:09 +0800 Subject: [PATCH] YARN-8103. Add CLI interface to query node attributes. Contributed by Bibin A Chundatt. --- .../hadoop/yarn/sls/nodemanager/NodeInfo.java | 12 +- .../yarn/sls/scheduler/RMNodeWrapper.java | 8 +- hadoop-yarn-project/hadoop-yarn/bin/yarn | 4 +- .../hadoop/yarn/api/records/NodeReport.java | 13 + .../src/main/proto/yarn_protos.proto | 1 + .../hadoop/yarn/client/cli/ClusterCLI.java | 17 + .../yarn/client/cli/NodeAttributesCLI.java | 929 ++++++++++++------ .../hadoop/yarn/client/cli/NodeCLI.java | 13 +- .../yarn/client/cli/TestClusterCLI.java | 32 +- .../client/cli/TestNodeAttributesCLI.java | 331 +++++-- .../hadoop/yarn/client/cli/TestYarnCLI.java | 31 +- .../impl/pb/NodeAttributeInfoPBImpl.java | 10 +- .../records/impl/pb/NodeAttributePBImpl.java | 12 +- .../api/records/impl/pb/NodeReportPBImpl.java | 44 +- .../yarn/server/utils/BuilderUtils.java | 6 +- .../server/resourcemanager/AdminService.java | 5 +- .../resourcemanager/ClientRMService.java | 5 +- .../ResourceTrackerService.java | 4 - .../server/resourcemanager/rmnode/RMNode.java | 11 +- .../resourcemanager/rmnode/RMNodeImpl.java | 18 +- .../resourcemanager/webapp/dao/NodeInfo.java | 13 +- .../server/resourcemanager/MockNodes.java | 4 +- .../resourcemanager/TestRMAdminService.java | 14 +- 23 files changed, 1067 insertions(+), 470 deletions(-) diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java index 65b8da0ff5..2eee351717 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.sls.nodemanager; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -220,16 +221,9 @@ public Map getAllocationTagsWithCount() { return null; } - @Override - public void setNodeAttributes(String prefix, - Set nodeAttributes) { - - } - - @Override - public Map> getAllNodeAttributes() { - return null; + public Set getAllNodeAttributes() { + return Collections.emptySet(); } @Override diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java index bf61f547ec..248b634b17 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java @@ -209,13 +209,7 @@ public Map getAllocationTagsWithCount() { } @Override - public void setNodeAttributes(String prefix, - Set nodeAttributes) { - node.setNodeAttributes(prefix, nodeAttributes); - } - - @Override - public Map> getAllNodeAttributes() { + public Set getAllNodeAttributes() { return node.getAllNodeAttributes(); } diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index 7cd838fb81..8290fcda8d 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -55,7 +55,7 @@ function hadoop_usage hadoop_add_subcommand "timelinereader" client "run the timeline reader server" hadoop_add_subcommand "timelineserver" daemon "run the timeline server" hadoop_add_subcommand "top" client "view cluster information" - hadoop_add_subcommand "node-attributes" "map node to attibutes" + hadoop_add_subcommand "nodeattributes" client "node attributes cli client" hadoop_add_subcommand "version" client "print the version" hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" true } @@ -187,7 +187,7 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}" hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*" HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer' ;; - node-attributes) + nodeattributes) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="false" HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.cli.NodeAttributesCLI' ;; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java index 3a80641bb6..625ad23408 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java @@ -258,4 +258,17 @@ public NodeUpdateType getNodeUpdateType() { * Set the node update type (null indicates absent node update type). * */ public void setNodeUpdateType(NodeUpdateType nodeUpdateType) {} + + /** + * Set the node attributes of node. + * + * @param nodeAttributes set of node attributes. + */ + public abstract void setNodeAttributes(Set nodeAttributes); + + /** + * Get node attributes of node. + * @return the set of node attributes. + */ + public abstract Set getNodeAttributes(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index aca9471f6f..10b36c75f5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -355,6 +355,7 @@ message NodeReportProto { optional ResourceUtilizationProto node_utilization = 12; optional uint32 decommissioning_timeout = 13; optional NodeUpdateTypeProto node_update_type = 14; + repeated NodeAttributeProto node_attributes = 15; } message NodeIdToLabelsProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ClusterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ClusterCLI.java index a29b0db736..4d93949845 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ClusterCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ClusterCLI.java @@ -36,6 +36,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.yarn.api.records.NodeAttributeInfo; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -52,6 +53,7 @@ public class ClusterCLI extends YarnCLI { public static final String LIST_LABELS_CMD = "list-node-labels"; public static final String DIRECTLY_ACCESS_NODE_LABEL_STORE = "directly-access-node-label-store"; + public static final String LIST_CLUSTER_ATTRIBUTES="list-node-attributes"; public static final String CMD = "cluster"; private boolean accessLocal = false; static CommonNodeLabelsManager localNodeLabelsManager = null; @@ -71,6 +73,8 @@ public int run(String[] args) throws Exception { opts.addOption("lnl", LIST_LABELS_CMD, false, "List cluster node-label collection"); + opts.addOption("lna", LIST_CLUSTER_ATTRIBUTES, false, + "List cluster node-attribute collection"); opts.addOption("h", HELP_CMD, false, "Displays help for all commands."); opts.addOption("dnl", DIRECTLY_ACCESS_NODE_LABEL_STORE, false, "This is DEPRECATED, will be removed in future releases. Directly access node label store, " @@ -102,6 +106,8 @@ public int run(String[] args) throws Exception { if (parsedCli.hasOption(LIST_LABELS_CMD)) { printClusterNodeLabels(); + } else if(parsedCli.hasOption(LIST_CLUSTER_ATTRIBUTES)){ + printClusterNodeAttributes(); } else if (parsedCli.hasOption(HELP_CMD)) { printUsage(opts); return 0; @@ -112,6 +118,17 @@ public int run(String[] args) throws Exception { return 0; } + private void printClusterNodeAttributes() throws IOException, YarnException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter( + new OutputStreamWriter(baos, Charset.forName("UTF-8"))); + for (NodeAttributeInfo attribute : client.getClusterAttributes()) { + pw.println(attribute.toString()); + } + pw.close(); + sysout.println(baos.toString("UTF-8")); + } + void printClusterNodeLabels() throws YarnException, IOException { List nodeLabels = null; if (accessLocal) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java index df5a57dff7..13d5e24c1c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java @@ -18,29 +18,30 @@ package org.apache.hadoop.yarn.client.cli; -import java.io.IOException; -import java.io.PrintStream; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.MissingArgumentException; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionGroup; import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.UnrecognizedOptionException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.CommonConfigurationKeys; -import org.apache.hadoop.ha.HAAdmin.UsageInfo; -import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesResponse; import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.NodeAttributeInfo; +import org.apache.hadoop.yarn.api.records.NodeAttributeKey; import org.apache.hadoop.yarn.api.records.NodeAttributeType; import org.apache.hadoop.yarn.client.ClientRMProxy; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -50,13 +51,24 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NodeToAttributes; import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingRequest; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintStream; +import java.io.PrintWriter; +import java.io.UnsupportedEncodingException; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; /** * CLI to map attributes to Nodes. - * */ public class NodeAttributesCLI extends Configured implements Tool { @@ -64,351 +76,640 @@ public class NodeAttributesCLI extends Configured implements Tool { "Invalid Node to attribute mapping : "; protected static final String USAGE_YARN_NODE_ATTRIBUTES = - "Usage: yarn node-attributes "; + "Usage: yarn nodeattributes "; + + protected static final String MISSING_ARGUMENT = + "Missing argument for command"; protected static final String NO_MAPPING_ERR_MSG = "No node-to-attributes mappings are specified"; - protected final static Map NODE_ATTRIB_USAGE = - ImmutableMap.builder() - .put("-replace", - new UsageInfo( - "<\"node1:attribute[(type)][=value],attribute1[=value]," - + "attribute2 node2:attribute2[=value],attribute3\">", - " Replace the node to attributes mapping information at the" - + " ResourceManager with the new mapping. Currently" - + " supported attribute type. And string is the default" - + " type too. Attribute value if not specified for string" - + " type value will be considered as empty string." - + " Replaced node-attributes should not violate the" - + " existing attribute to attribute type mapping.")) - .put("-add", - new UsageInfo( - "<\"node1:attribute[(type)][=value],attribute1[=value]," - + "attribute2 node2:attribute2[=value],attribute3\">", - " Adds or updates the node to attributes mapping information" - + " at the ResourceManager. Currently supported attribute" - + " type is string. And string is the default type too." - + " Attribute value if not specified for string type" - + " value will be considered as empty string. Added or" - + " updated node-attributes should not violate the" - + " existing attribute to attribute type mapping.")) - .put("-remove", - new UsageInfo("<\"node1:attribute,attribute1 node2:attribute2\">", - " Removes the specified node to attributes mapping" - + " information at the ResourceManager")) - .put("-failOnUnknownNodes", - new UsageInfo("", - "Can be used optionally along with other options. When its" - + " set, it will fail if specified nodes are unknown.")) - .build(); - - /** Output stream for errors, for use in tests. */ + private static final String DEFAULT_SEPARATOR = System.lineSeparator(); + public static final String INVALID_COMMAND_USAGE = "Invalid Command Usage : "; + /** + * Output stream for errors, for use in tests. + */ private PrintStream errOut = System.err; public NodeAttributesCLI() { super(); } - public NodeAttributesCLI(Configuration conf) { - super(conf); - } - protected void setErrOut(PrintStream errOut) { this.errOut = errOut; } - private void printHelpMsg(String cmd) { - StringBuilder builder = new StringBuilder(); - UsageInfo usageInfo = null; - if (cmd != null && !(cmd.trim().isEmpty())) { - usageInfo = NODE_ATTRIB_USAGE.get(cmd); - } - if (usageInfo != null) { - if (usageInfo.args == null) { - builder.append(" " + cmd + ":\n" + usageInfo.help); - } else { - String space = (usageInfo.args == "") ? "" : " "; - builder.append( - " " + cmd + space + usageInfo.args + " :\n" + usageInfo.help); - } - } else { - // help for all commands - builder.append("Usage: yarn node-attributes\n"); - for (Map.Entry cmdEntry : NODE_ATTRIB_USAGE - .entrySet()) { - usageInfo = cmdEntry.getValue(); - builder.append(" " + cmdEntry.getKey() + " " + usageInfo.args - + " :\n " + usageInfo.help + "\n"); - } - builder.append(" -help" + " [cmd]\n"); - } - errOut.println(builder); + protected AdminCommandHandler getAdminCommandHandler() { + return new AdminCommandHandler(); } - private static void buildIndividualUsageMsg(String cmd, - StringBuilder builder) { - UsageInfo usageInfo = NODE_ATTRIB_USAGE.get(cmd); - if (usageInfo == null) { - return; - } - if (usageInfo.args == null) { - builder.append(USAGE_YARN_NODE_ATTRIBUTES + cmd + "\n"); - } else { - String space = (usageInfo.args == "") ? "" : " "; - builder.append( - USAGE_YARN_NODE_ATTRIBUTES + cmd + space + usageInfo.args + "\n"); - } + protected ClientCommandHandler getClientCommandHandler() { + return new ClientCommandHandler(); } - private static void buildUsageMsgForAllCmds(StringBuilder builder) { - builder.append("Usage: yarn node-attributes\n"); - for (Map.Entry cmdEntry : NODE_ATTRIB_USAGE.entrySet()) { - UsageInfo usageInfo = cmdEntry.getValue(); - builder.append(" " + cmdEntry.getKey() + " " + usageInfo.args + "\n"); - } - builder.append(" -help" + " [cmd]\n"); - } - - /** - * Displays format of commands. - * - * @param cmd The command that is being executed. - */ - private void printUsage(String cmd) { + void printUsage(String cmd, boolean desc, CommandHandler... handlers) + throws UnsupportedEncodingException { StringBuilder usageBuilder = new StringBuilder(); - if (NODE_ATTRIB_USAGE.containsKey(cmd)) { - buildIndividualUsageMsg(cmd, usageBuilder); + usageBuilder.append(USAGE_YARN_NODE_ATTRIBUTES); + boolean satisfied = false; + for (CommandHandler cmdHandlers : handlers) { + satisfied |= cmdHandlers.getHelp(cmd, usageBuilder, desc); + } + if (!satisfied) { + printUsage(desc, handlers); } else { - buildUsageMsgForAllCmds(usageBuilder); + print(usageBuilder); } - errOut.println(usageBuilder); } - private void printUsage() { - printUsage(""); - } - - protected ResourceManagerAdministrationProtocol createAdminProtocol() - throws IOException { - // Get the current configuration - final YarnConfiguration conf = new YarnConfiguration(getConf()); - return ClientRMProxy.createRMProxy(conf, - ResourceManagerAdministrationProtocol.class); - } - - @Override - public void setConf(Configuration conf) { - if (conf != null) { - conf = addSecurityConfiguration(conf); + private void printUsage(boolean desc, CommandHandler... handlers) + throws UnsupportedEncodingException { + StringBuilder usageBuilder = new StringBuilder(); + usageBuilder.append(USAGE_YARN_NODE_ATTRIBUTES); + for (CommandHandler cmdHandlers : handlers) { + cmdHandlers.getHelp(usageBuilder, desc); } - super.setConf(conf); + + // append help with usage + usageBuilder.append(DEFAULT_SEPARATOR) + .append(" -help [cmd] List help of commands"); + print(usageBuilder); } - /** - * Add the requisite security principal settings to the given Configuration, - * returning a copy. - * - * @param conf the original config - * @return a copy with the security settings added - */ - private static Configuration addSecurityConfiguration(Configuration conf) { - // Make a copy so we don't mutate it. Also use an YarnConfiguration to - // force loading of yarn-site.xml. - conf = new YarnConfiguration(conf); - conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY, - conf.get(YarnConfiguration.RM_PRINCIPAL, "")); - return conf; + private void print(StringBuilder usageBuilder) + throws UnsupportedEncodingException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = + new PrintWriter(new OutputStreamWriter(baos, Charset.forName("UTF-8"))); + pw.write(usageBuilder.toString()); + pw.close(); + errOut.println(baos.toString("UTF-8")); + } + + private Options buildOptions(CommandHandler... handlers) { + Options opts = new Options(); + for (CommandHandler handler : handlers) { + Options handlerOpts = handler.getOptions(); + handlerOpts.getOptions().iterator() + .forEachRemaining(option -> opts.addOption((Option) option)); + } + return opts; } - @Override public int run(String[] args) throws Exception { + + int exitCode = -1; + + AdminCommandHandler adminCmdHandler = getAdminCommandHandler(); + ClientCommandHandler clientCmdHandler = getClientCommandHandler(); + + // Build options + Options opts = buildOptions(adminCmdHandler, clientCmdHandler); + if (args.length < 1) { - printUsage(); + printUsage(false, adminCmdHandler, clientCmdHandler); return -1; } - int exitCode = -1; - int i = 0; - String cmd = args[i++]; - - if ("-help".equals(cmd)) { - exitCode = 0; - if (args.length >= 2) { - printHelpMsg(args[i]); - } else { - printHelpMsg(""); - } - return exitCode; + // Handle command separate + if (handleHelpCommand(args, adminCmdHandler, clientCmdHandler)) { + return 0; } - try { - if ("-replace".equals(cmd)) { - exitCode = handleNodeAttributeMapping(args, - AttributeMappingOperationType.REPLACE); - } else if ("-add".equals(cmd)) { - exitCode = - handleNodeAttributeMapping(args, AttributeMappingOperationType.ADD); - } else if ("-remove".equals(cmd)) { - exitCode = handleNodeAttributeMapping(args, - AttributeMappingOperationType.REMOVE); - } else { - exitCode = -1; - errOut.println(cmd.substring(1) + ": Unknown command"); - printUsage(); - } - } catch (IllegalArgumentException arge) { - exitCode = -1; - errOut.println(cmd.substring(1) + ": " + arge.getLocalizedMessage()); - printUsage(cmd); - } catch (RemoteException e) { - // - // This is a error returned by hadoop server. Print - // out the first line of the error message, ignore the stack trace. - exitCode = -1; - try { - String[] content; - content = e.getLocalizedMessage().split("\n"); - errOut.println(cmd.substring(1) + ": " + content[0]); - } catch (Exception ex) { - errOut.println(cmd.substring(1) + ": " + ex.getLocalizedMessage()); - } - } catch (Exception e) { - exitCode = -1; - errOut.println(cmd.substring(1) + ": " + e.getLocalizedMessage()); - } - return exitCode; - } - - private int handleNodeAttributeMapping(String args[], - AttributeMappingOperationType operation) - throws IOException, YarnException, ParseException { - Options opts = new Options(); - opts.addOption(operation.name().toLowerCase(), true, - operation.name().toLowerCase()); - opts.addOption("failOnUnknownNodes", false, "Fail on unknown nodes."); - int exitCode = -1; - CommandLine cliParser = null; + CommandLine cliParser; + CommandHandler handler = null; try { cliParser = new GnuParser().parse(opts, args); + handler = adminCmdHandler.canHandleCommand(cliParser) ? + adminCmdHandler : + clientCmdHandler.canHandleCommand(cliParser) ? + clientCmdHandler : + null; + if (handler == null) { + errOut.println(INVALID_COMMAND_USAGE); + printUsage(false, adminCmdHandler, clientCmdHandler); + return exitCode; + } else { + return handler.handleCommand(cliParser); + } + } catch (UnrecognizedOptionException e) { + errOut.println(INVALID_COMMAND_USAGE); + printUsage(false, adminCmdHandler, clientCmdHandler); + return exitCode; } catch (MissingArgumentException ex) { - errOut.println(NO_MAPPING_ERR_MSG); - printUsage(args[0]); + errOut.println(MISSING_ARGUMENT); + printUsage(true, adminCmdHandler, clientCmdHandler); + return exitCode; + } catch (IllegalArgumentException arge) { + errOut.println(arge.getLocalizedMessage()); + // print admin command detail + printUsage(true, handler); + return exitCode; + } catch (Exception e) { + errOut.println(e.toString()); + printUsage(true, handler); return exitCode; } - List buildNodeLabelsMapFromStr = - buildNodeLabelsMapFromStr( - cliParser.getOptionValue(operation.name().toLowerCase()), - operation != AttributeMappingOperationType.REPLACE, operation); - NodesToAttributesMappingRequest request = NodesToAttributesMappingRequest - .newInstance(operation, buildNodeLabelsMapFromStr, - cliParser.hasOption("failOnUnknownNodes")); - ResourceManagerAdministrationProtocol adminProtocol = createAdminProtocol(); - adminProtocol.mapAttributesToNodes(request); - return 0; } - /** - * args are expected to be of the format - * node1:java(string)=8,ssd(boolean)=false node2:ssd(boolean)=true - */ - private List buildNodeLabelsMapFromStr(String args, - boolean validateForAttributes, AttributeMappingOperationType operation) { - Map nodeToAttributesMap = new HashMap<>(); - for (String nodeToAttributesStr : args.split("[ \n]")) { - // for each node to attribute mapping - nodeToAttributesStr = nodeToAttributesStr.trim(); - if (nodeToAttributesStr.isEmpty() - || nodeToAttributesStr.startsWith("#")) { - continue; + private boolean handleHelpCommand(String[] args, CommandHandler... handlers) + throws UnsupportedEncodingException { + if (args[0].equals("-help")) { + if (args.length == 2) { + printUsage(args[1], true, handlers); + } else { + printUsage(true, handlers); } - if (nodeToAttributesStr.indexOf(":") == -1) { - throw new IllegalArgumentException( - INVALID_MAPPING_ERR_MSG + nodeToAttributesStr); - } - String[] nodeToAttributes = nodeToAttributesStr.split(":"); - Preconditions.checkArgument(!nodeToAttributes[0].trim().isEmpty(), - "Node name cannot be empty"); - String node = nodeToAttributes[0]; - String[] attributeNameValueType = null; - List attributesList = new ArrayList<>(); - NodeAttributeType attributeType = NodeAttributeType.STRING; - String attributeValue; - String attributeName; - Set attributeNamesMapped = new HashSet<>(); - - String attributesStr[]; - if (nodeToAttributes.length == 2) { - // fetching multiple attributes for a node - attributesStr = nodeToAttributes[1].split(","); - for (String attributeStr : attributesStr) { - // get information about each attribute. - attributeNameValueType = attributeStr.split("="); // to find name - // value - Preconditions.checkArgument( - !(attributeNameValueType[0] == null - || attributeNameValueType[0].isEmpty()), - "Attribute name cannot be null or empty"); - attributeValue = attributeNameValueType.length > 1 - ? attributeNameValueType[1] : ""; - int indexOfOpenBracket = attributeNameValueType[0].indexOf("("); - if (indexOfOpenBracket == -1) { - attributeName = attributeNameValueType[0]; - } else if (indexOfOpenBracket == 0) { - throw new IllegalArgumentException("Attribute for node " + node - + " is not properly configured : " + attributeStr); - } else { - // attribute type has been explicitly configured - int indexOfCloseBracket = attributeNameValueType[0].indexOf(")"); - if (indexOfCloseBracket == -1 - || indexOfCloseBracket < indexOfOpenBracket) { - throw new IllegalArgumentException("Attribute for node " + node - + " is not properly Configured : " + attributeStr); - } - String attributeTypeStr; - attributeName = - attributeNameValueType[0].substring(0, indexOfOpenBracket); - attributeTypeStr = attributeNameValueType[0] - .substring(indexOfOpenBracket + 1, indexOfCloseBracket); - try { - attributeType = NodeAttributeType - .valueOf(attributeTypeStr.trim().toUpperCase()); - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException( - "Invalid Attribute type configuration : " + attributeTypeStr - + " in " + attributeStr); - } - } - if (attributeNamesMapped.contains(attributeName)) { - throw new IllegalArgumentException("Attribute " + attributeName - + " has been mapped more than once in : " - + nodeToAttributesStr); - } - // TODO when we support different type of attribute type we need to - // cross verify whether input attributes itself is not violating - // attribute Name to Type mapping. - attributesList - .add(NodeAttribute.newInstance(NodeAttribute.PREFIX_CENTRALIZED, - attributeName.trim(), attributeType, attributeValue.trim())); - } - } - if (validateForAttributes) { - Preconditions.checkArgument((attributesList.size() > 0), - "Attributes cannot be null or empty for Operation " - + operation.name() + " on the node " + node); - } - nodeToAttributesMap - .put(node,NodeToAttributes.newInstance(node, attributesList)); + return true; } - - if (nodeToAttributesMap.isEmpty()) { - throw new IllegalArgumentException(NO_MAPPING_ERR_MSG); - } - return Lists.newArrayList(nodeToAttributesMap.values()); + return false; } public static void main(String[] args) throws Exception { int result = ToolRunner.run(new NodeAttributesCLI(), args); System.exit(result); } + + /** + * Abstract class for command handler. + */ + public static abstract class CommandHandler extends Configured { + + private Options options; + + private LinkedList order = new LinkedList<>(); + private String header; + + protected CommandHandler(String header) { + this(new YarnConfiguration()); + this.header = header; + } + + protected CommandHandler(Configuration conf) { + super(conf); + options = buildOptions(); + } + + public boolean canHandleCommand(CommandLine parse) { + ArrayList