YARN-11593. [Federation] Improve command line help information. (#6199) Contributed by Shilun Fan.
Reviewed-by: Inigo Goiri <inigoiri@apache.org> Signed-off-by: Shilun Fan <slfan1989@apache.org>
This commit is contained in:
parent
8bd1f65efc
commit
d18410221b
@ -23,7 +23,10 @@
|
|||||||
import org.apache.commons.cli.Option;
|
import org.apache.commons.cli.Option;
|
||||||
import org.apache.commons.cli.Options;
|
import org.apache.commons.cli.Options;
|
||||||
import org.apache.commons.cli.ParseException;
|
import org.apache.commons.cli.ParseException;
|
||||||
|
import org.apache.commons.collections.CollectionUtils;
|
||||||
|
import org.apache.commons.collections.MapUtils;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.apache.hadoop.classification.VisibleForTesting;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.conf.Configured;
|
import org.apache.hadoop.conf.Configured;
|
||||||
import org.apache.hadoop.ha.HAAdmin.UsageInfo;
|
import org.apache.hadoop.ha.HAAdmin.UsageInfo;
|
||||||
@ -67,6 +70,7 @@
|
|||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
import static org.apache.hadoop.yarn.server.api.protocolrecords.FederationQueueWeight.checkHeadRoomAlphaValid;
|
import static org.apache.hadoop.yarn.server.api.protocolrecords.FederationQueueWeight.checkHeadRoomAlphaValid;
|
||||||
@ -74,29 +78,8 @@
|
|||||||
|
|
||||||
public class RouterCLI extends Configured implements Tool {
|
public class RouterCLI extends Configured implements Tool {
|
||||||
|
|
||||||
|
|
||||||
private static final Logger LOG = LoggerFactory.getLogger(RouterCLI.class);
|
private static final Logger LOG = LoggerFactory.getLogger(RouterCLI.class);
|
||||||
|
|
||||||
protected final static Map<String, UsageInfo> ADMIN_USAGE =
|
|
||||||
ImmutableMap.<String, UsageInfo>builder()
|
|
||||||
// Command1: deregisterSubCluster
|
|
||||||
.put("-deregisterSubCluster", new UsageInfo(
|
|
||||||
"[-sc|--subClusterId [subCluster Id]]",
|
|
||||||
"Deregister SubCluster, If the interval between the heartbeat time of the subCluster " +
|
|
||||||
"and the current time exceeds the timeout period, " +
|
|
||||||
"set the state of the subCluster to SC_LOST."))
|
|
||||||
// Command2: policy
|
|
||||||
.put("-policy", new UsageInfo(
|
|
||||||
"[-s|--save [queue;router weight;amrm weight;headroomalpha]] " +
|
|
||||||
"[-bs|--batch-save [--format xml] [-f|--input-file fileName]]" +
|
|
||||||
"[-l|--list [--pageSize][--currentPage][--queue][--queues]]",
|
|
||||||
"We provide a set of commands for Policy:" +
|
|
||||||
" Include list policies, save policies, batch save policies. " +
|
|
||||||
" (Note: The policy type will be directly read from the" +
|
|
||||||
" yarn.federation.policy-manager in the local yarn-site.xml.)" +
|
|
||||||
" eg. (routeradmin -policy [-s|--save] root.a;SC-1:0.7,SC-2:0.3;SC-1:0.7,SC-2:0.3;1.0)"))
|
|
||||||
.build();
|
|
||||||
|
|
||||||
// Common Constant
|
// Common Constant
|
||||||
private static final String SEMICOLON = ";";
|
private static final String SEMICOLON = ";";
|
||||||
|
|
||||||
@ -104,6 +87,7 @@ public class RouterCLI extends Configured implements Tool {
|
|||||||
private static final String CMD_EMPTY = "";
|
private static final String CMD_EMPTY = "";
|
||||||
private static final int EXIT_SUCCESS = 0;
|
private static final int EXIT_SUCCESS = 0;
|
||||||
private static final int EXIT_ERROR = -1;
|
private static final int EXIT_ERROR = -1;
|
||||||
|
private static final String CMD_HELP = "-help";
|
||||||
|
|
||||||
// Command1: deregisterSubCluster
|
// Command1: deregisterSubCluster
|
||||||
private static final String DEREGISTER_SUBCLUSTER_TITLE =
|
private static final String DEREGISTER_SUBCLUSTER_TITLE =
|
||||||
@ -115,17 +99,46 @@ public class RouterCLI extends Configured implements Tool {
|
|||||||
private static final String OPTION_SC = "sc";
|
private static final String OPTION_SC = "sc";
|
||||||
private static final String OPTION_SUBCLUSTERID = "subClusterId";
|
private static final String OPTION_SUBCLUSTERID = "subClusterId";
|
||||||
private static final String CMD_DEREGISTERSUBCLUSTER = "-deregisterSubCluster";
|
private static final String CMD_DEREGISTERSUBCLUSTER = "-deregisterSubCluster";
|
||||||
private static final String CMD_HELP = "-help";
|
|
||||||
|
// DeregisterSubCluster Command Parameters
|
||||||
|
protected final static UsageInfo SUBCLUSTER_ID = new UsageInfo("<-sc|--subClusterId>",
|
||||||
|
"'-sc' option allows you to specify the sub-cluster to operate on, " +
|
||||||
|
"while the '--subClusterId' option is the long format of -sc and serves the same purpose.");
|
||||||
|
|
||||||
|
// DeregisterSubCluster Command Examples
|
||||||
|
protected final static String DEREGISTER_SUBCLUSTER_EXAMPLE_1 =
|
||||||
|
"yarn routeradmin -deregisterSubCluster -sc SC-1";
|
||||||
|
protected final static String DEREGISTER_SUBCLUSTER_EXAMPLE_2 =
|
||||||
|
"yarn routeradmin -deregisterSubCluster --subClusterId SC-1";
|
||||||
|
|
||||||
|
// DeregisterSubCluster Command Help Information
|
||||||
|
protected final static String DEREGISTER_SUBCLUSTER_HELP_INFO =
|
||||||
|
"deregister subCluster, If the interval between the heartbeat time of the subCluster and" +
|
||||||
|
"the current time exceeds the timeout period, set the state of the subCluster to SC_LOST.";
|
||||||
|
|
||||||
|
protected final static RouterCmdUsageInfos DEREGISTER_SUBCLUSTER_USAGEINFOS =
|
||||||
|
new RouterCmdUsageInfos()
|
||||||
|
.addUsageInfo(SUBCLUSTER_ID)
|
||||||
|
.addHelpInfo(DEREGISTER_SUBCLUSTER_HELP_INFO)
|
||||||
|
.addExampleDescs(CMD_DEREGISTERSUBCLUSTER, "If we want to deregisterSubCluster SC-1")
|
||||||
|
.addExample(CMD_DEREGISTERSUBCLUSTER, DEREGISTER_SUBCLUSTER_EXAMPLE_1)
|
||||||
|
.addExample(CMD_DEREGISTERSUBCLUSTER, DEREGISTER_SUBCLUSTER_EXAMPLE_2);
|
||||||
|
|
||||||
// Command2: policy
|
// Command2: policy
|
||||||
|
|
||||||
|
private static final String CMD_POLICY = "-policy";
|
||||||
|
|
||||||
// save policy
|
// save policy
|
||||||
private static final String OPTION_S = "s";
|
private static final String OPTION_S = "s";
|
||||||
private static final String OPTION_BATCH_S = "bs";
|
|
||||||
private static final String OPTION_SAVE = "save";
|
private static final String OPTION_SAVE = "save";
|
||||||
|
// batch save policy
|
||||||
|
private static final String OPTION_BATCH_S = "bs";
|
||||||
private static final String OPTION_BATCH_SAVE = "batch-save";
|
private static final String OPTION_BATCH_SAVE = "batch-save";
|
||||||
private static final String OPTION_FORMAT = "format";
|
private static final String OPTION_FORMAT = "format";
|
||||||
|
private static final String FORMAT_XML = "xml";
|
||||||
private static final String OPTION_FILE = "f";
|
private static final String OPTION_FILE = "f";
|
||||||
private static final String OPTION_INPUT_FILE = "input-file";
|
private static final String OPTION_INPUT_FILE = "input-file";
|
||||||
|
// list policy
|
||||||
private static final String OPTION_L = "l";
|
private static final String OPTION_L = "l";
|
||||||
private static final String OPTION_LIST = "list";
|
private static final String OPTION_LIST = "list";
|
||||||
private static final String OPTION_PAGE_SIZE = "pageSize";
|
private static final String OPTION_PAGE_SIZE = "pageSize";
|
||||||
@ -133,9 +146,6 @@ public class RouterCLI extends Configured implements Tool {
|
|||||||
private static final String OPTION_QUEUE = "queue";
|
private static final String OPTION_QUEUE = "queue";
|
||||||
private static final String OPTION_QUEUES = "queues";
|
private static final String OPTION_QUEUES = "queues";
|
||||||
|
|
||||||
private static final String CMD_POLICY = "-policy";
|
|
||||||
private static final String FORMAT_XML = "xml";
|
|
||||||
private static final String FORMAT_JSON = "json";
|
|
||||||
private static final String XML_TAG_SUBCLUSTERIDINFO = "subClusterIdInfo";
|
private static final String XML_TAG_SUBCLUSTERIDINFO = "subClusterIdInfo";
|
||||||
private static final String XML_TAG_AMRMPOLICYWEIGHTS = "amrmPolicyWeights";
|
private static final String XML_TAG_AMRMPOLICYWEIGHTS = "amrmPolicyWeights";
|
||||||
private static final String XML_TAG_ROUTERPOLICYWEIGHTS = "routerPolicyWeights";
|
private static final String XML_TAG_ROUTERPOLICYWEIGHTS = "routerPolicyWeights";
|
||||||
@ -146,10 +156,85 @@ public class RouterCLI extends Configured implements Tool {
|
|||||||
|
|
||||||
private static final String LIST_POLICIES_TITLE =
|
private static final String LIST_POLICIES_TITLE =
|
||||||
"Yarn Federation Queue Policies";
|
"Yarn Federation Queue Policies";
|
||||||
|
|
||||||
// Columns information
|
// Columns information
|
||||||
private static final List<String> LIST_POLICIES_HEADER = Arrays.asList(
|
private static final List<String> LIST_POLICIES_HEADER = Arrays.asList(
|
||||||
"Queue Name", "AMRM Weight", "Router Weight");
|
"Queue Name", "AMRM Weight", "Router Weight");
|
||||||
|
|
||||||
|
// Policy Commands
|
||||||
|
protected final static UsageInfo POLICY_SAVE_USAGE = new UsageInfo(
|
||||||
|
"-s|--save (<queue;router weight;amrm weight;headroomalpha>)",
|
||||||
|
"This command is used to save the policy information of the queue, " +
|
||||||
|
"including queue and weight information.");
|
||||||
|
|
||||||
|
protected final static String POLICY_SAVE_USAGE_EXAMPLE_DESC =
|
||||||
|
"We have two sub-clusters, SC-1 and SC-2. \\" +
|
||||||
|
"We want to configure a weight policy for the 'root.a' queue. \\" +
|
||||||
|
"The Router Weight is set to SC-1 with a weight of 0.7 and SC-2 with a weight of 0.3. \\" +
|
||||||
|
"The AMRM Weight is set SC-1 to 0.6 and SC-2 to 0.4. \\" +
|
||||||
|
"We are using the default value of 0.1 for headroomalpha.";
|
||||||
|
|
||||||
|
protected final static String POLICY_SAVE_USAGE_EXAMPLE_1 =
|
||||||
|
"yarn routeradmin -policy -s root.a;SC-1:0.7,SC-2:0.3;SC-1:0.6,SC-2:0.4;1.0";
|
||||||
|
protected final static String POLICY_SAVE_USAGE_EXAMPLE_2 =
|
||||||
|
"yarn routeradmin -policy --save root.a;SC-1:0.7,SC-2:0.3;SC-1:0.6,SC-2:0.4;1.0";
|
||||||
|
|
||||||
|
protected final static UsageInfo POLICY_BATCH_SAVE_USAGE = new UsageInfo(
|
||||||
|
"-bs|--batch-save (--format <xml>) (-f|--input-file <fileName>)",
|
||||||
|
"This command can batch load weight information for queues " +
|
||||||
|
"based on the provided `federation-weights.xml` file.");
|
||||||
|
|
||||||
|
protected final static String POLICY_BATCH_SAVE_USAGE_EXAMPLE_DESC =
|
||||||
|
"We have two sub-clusters, SC-1 and SC-2. \\" +
|
||||||
|
"We would like to configure weights for 'root.a' and 'root.b' queues. \\" +
|
||||||
|
"We can set the weights for 'root.a' and 'root.b' in the 'federation-weights.xml' file. \\" +
|
||||||
|
"and then use the batch-save command to save the configurations in bulk.";
|
||||||
|
|
||||||
|
protected final static String POLICY_BATCH_SAVE_USAGE_EXAMPLE_1 =
|
||||||
|
"yarn routeradmin -policy -bs --format xml -f federation-weights.xml";
|
||||||
|
protected final static String POLICY_BATCH_SAVE_USAGE_EXAMPLE_2 =
|
||||||
|
"yarn routeradmin -policy --batch-save --format xml -f federation-weights.xml";
|
||||||
|
|
||||||
|
protected final static UsageInfo POLICY_LIST_USAGE = new UsageInfo(
|
||||||
|
"-l|--list [--pageSize][--currentPage][--queue][--queues]",
|
||||||
|
"This command is used to display the configured queue weight information.");
|
||||||
|
|
||||||
|
protected final static String POLICY_LIST_USAGE_EXAMPLE_DESC =
|
||||||
|
"We can display the list of already configured queue weight information. \\" +
|
||||||
|
"We can use the --queue option to query the weight information for a specific queue \\" +
|
||||||
|
" or use the --queues option to query the weight information for multiple queues. \\";
|
||||||
|
|
||||||
|
protected final static String POLICY_LIST_USAGE_EXAMPLE_1 =
|
||||||
|
"yarn routeradmin -policy -l --pageSize 20 --currentPage 1 --queue root.a";
|
||||||
|
|
||||||
|
protected final static String POLICY_LIST_USAGE_EXAMPLE_2 =
|
||||||
|
"yarn routeradmin -policy -list --pageSize 20 --currentPage 1 --queues root.a,root.b";
|
||||||
|
|
||||||
|
protected final static RouterCmdUsageInfos POLICY_USAGEINFOS = new RouterCmdUsageInfos()
|
||||||
|
// Policy Save
|
||||||
|
.addUsageInfo(POLICY_SAVE_USAGE)
|
||||||
|
.addExampleDescs(POLICY_SAVE_USAGE.args, POLICY_SAVE_USAGE_EXAMPLE_DESC)
|
||||||
|
.addExample(POLICY_SAVE_USAGE.args, POLICY_SAVE_USAGE_EXAMPLE_1)
|
||||||
|
.addExample(POLICY_SAVE_USAGE.args, POLICY_SAVE_USAGE_EXAMPLE_2)
|
||||||
|
// Policy Batch Save
|
||||||
|
.addUsageInfo(POLICY_BATCH_SAVE_USAGE)
|
||||||
|
.addExampleDescs(POLICY_BATCH_SAVE_USAGE.args, POLICY_BATCH_SAVE_USAGE_EXAMPLE_DESC)
|
||||||
|
.addExample(POLICY_BATCH_SAVE_USAGE.args, POLICY_BATCH_SAVE_USAGE_EXAMPLE_1)
|
||||||
|
.addExample(POLICY_BATCH_SAVE_USAGE.args, POLICY_BATCH_SAVE_USAGE_EXAMPLE_2)
|
||||||
|
// Policy List Save
|
||||||
|
.addUsageInfo(POLICY_LIST_USAGE)
|
||||||
|
.addExampleDescs(POLICY_LIST_USAGE.args, POLICY_LIST_USAGE_EXAMPLE_DESC)
|
||||||
|
.addExample(POLICY_LIST_USAGE.args, POLICY_LIST_USAGE_EXAMPLE_1)
|
||||||
|
.addExample(POLICY_LIST_USAGE.args, POLICY_LIST_USAGE_EXAMPLE_2);
|
||||||
|
|
||||||
|
protected final static Map<String, RouterCmdUsageInfos> ADMIN_USAGE =
|
||||||
|
ImmutableMap.<String, RouterCmdUsageInfos>builder()
|
||||||
|
// Command1: deregisterSubCluster
|
||||||
|
.put(CMD_DEREGISTERSUBCLUSTER, DEREGISTER_SUBCLUSTER_USAGEINFOS)
|
||||||
|
// Command2: policy
|
||||||
|
.put(CMD_POLICY, POLICY_USAGEINFOS)
|
||||||
|
.build();
|
||||||
|
|
||||||
public RouterCLI() {
|
public RouterCLI() {
|
||||||
super();
|
super();
|
||||||
}
|
}
|
||||||
@ -159,43 +244,66 @@ public RouterCLI(Configuration conf) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private static void buildHelpMsg(String cmd, StringBuilder builder) {
|
private static void buildHelpMsg(String cmd, StringBuilder builder) {
|
||||||
UsageInfo usageInfo = ADMIN_USAGE.get(cmd);
|
RouterCmdUsageInfos routerUsageInfo = ADMIN_USAGE.get(cmd);
|
||||||
if (usageInfo == null) {
|
|
||||||
|
if (routerUsageInfo == null) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
builder.append("[").append(cmd).append("]\n");
|
||||||
|
|
||||||
if (usageInfo.args != null) {
|
if (!routerUsageInfo.helpInfos.isEmpty()) {
|
||||||
String space = (usageInfo.args == "") ? "" : " ";
|
builder.append("\t Description: \n");
|
||||||
builder.append(" ")
|
for (String helpInfo : routerUsageInfo.helpInfos) {
|
||||||
.append(cmd)
|
builder.append("\t\t").append(helpInfo).append("\n\n");
|
||||||
.append(space)
|
|
||||||
.append(usageInfo.args)
|
|
||||||
.append(": ")
|
|
||||||
.append(usageInfo.help);
|
|
||||||
} else {
|
|
||||||
builder.append(" ")
|
|
||||||
.append(cmd)
|
|
||||||
.append(": ")
|
|
||||||
.append(usageInfo.help);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void buildIndividualUsageMsg(String cmd, StringBuilder builder) {
|
if (!routerUsageInfo.usageInfos.isEmpty()) {
|
||||||
UsageInfo usageInfo = ADMIN_USAGE.get(cmd);
|
builder.append("\t UsageInfos: \n");
|
||||||
if (usageInfo == null) {
|
for (UsageInfo usageInfo : routerUsageInfo.usageInfos) {
|
||||||
return;
|
builder.append("\t\t").append(usageInfo.args)
|
||||||
|
.append(": ")
|
||||||
|
.append("\n\t\t")
|
||||||
|
.append(usageInfo.help).append("\n\n");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (MapUtils.isNotEmpty(routerUsageInfo.examples)) {
|
||||||
|
builder.append("\t Examples: \n");
|
||||||
|
int count = 1;
|
||||||
|
for (Map.Entry<String, List<String>> example : routerUsageInfo.examples.entrySet()) {
|
||||||
|
|
||||||
|
String keyCmd = example.getKey();
|
||||||
|
builder.append("\t\t")
|
||||||
|
.append("Cmd:").append(count)
|
||||||
|
.append(". ").append(keyCmd)
|
||||||
|
.append(": \n\n");
|
||||||
|
|
||||||
|
// Print Command Description
|
||||||
|
List<String> exampleDescs = routerUsageInfo.exampleDescs.get(keyCmd);
|
||||||
|
if (CollectionUtils.isNotEmpty(exampleDescs)) {
|
||||||
|
builder.append("\t\t").append("Cmd Requirement Description:\n");
|
||||||
|
for (String value : exampleDescs) {
|
||||||
|
String[] valueDescs = StringUtils.split(value, "\\");
|
||||||
|
for (String valueDesc : valueDescs) {
|
||||||
|
builder.append("\t\t").append(valueDesc).append("\n");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
builder.append("\n");
|
||||||
|
|
||||||
|
// Print Command example
|
||||||
|
List<String> valueExamples = example.getValue();
|
||||||
|
if (CollectionUtils.isNotEmpty(valueExamples)) {
|
||||||
|
builder.append("\t\t").append("Cmd Examples:\n");
|
||||||
|
for (String valueExample : valueExamples) {
|
||||||
|
builder.append("\t\t").append(valueExample).append("\n");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
builder.append("\n");
|
||||||
|
count++;
|
||||||
}
|
}
|
||||||
if (usageInfo.args == null) {
|
|
||||||
builder.append("Usage: routeradmin [")
|
|
||||||
.append(cmd)
|
|
||||||
.append("]\n");
|
|
||||||
} else {
|
|
||||||
String space = (usageInfo.args == "") ? "" : " ";
|
|
||||||
builder.append("Usage: routeradmin [")
|
|
||||||
.append(cmd)
|
|
||||||
.append(space)
|
|
||||||
.append(usageInfo.args)
|
|
||||||
.append("]\n");
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -204,12 +312,7 @@ private static void printHelp() {
|
|||||||
summary.append("routeradmin is the command to execute ")
|
summary.append("routeradmin is the command to execute ")
|
||||||
.append("YARN Federation administrative commands.\n")
|
.append("YARN Federation administrative commands.\n")
|
||||||
.append("The full syntax is: \n\n")
|
.append("The full syntax is: \n\n")
|
||||||
.append("routeradmin\n")
|
.append("routeradmin\n");
|
||||||
.append(" [-deregisterSubCluster [-sc|--subClusterId [subCluster Id]]\n")
|
|
||||||
.append(" [-policy [-s|--save [queue;router weight;amrm weight;headroomalpha] " +
|
|
||||||
"[-bs|--batch-save [--format xml,json] [-f|--input-file fileName]]] " +
|
|
||||||
"[-l|--list [--pageSize][--currentPage][--queue][--queues]]\n")
|
|
||||||
.append(" [-help [cmd]]").append("\n");
|
|
||||||
StringBuilder helpBuilder = new StringBuilder();
|
StringBuilder helpBuilder = new StringBuilder();
|
||||||
System.out.println(summary);
|
System.out.println(summary);
|
||||||
|
|
||||||
@ -235,13 +338,9 @@ protected ResourceManagerAdministrationProtocol createAdminProtocol()
|
|||||||
private static void buildUsageMsg(StringBuilder builder) {
|
private static void buildUsageMsg(StringBuilder builder) {
|
||||||
builder.append("routeradmin is only used in Yarn Federation Mode.\n");
|
builder.append("routeradmin is only used in Yarn Federation Mode.\n");
|
||||||
builder.append("Usage: routeradmin\n");
|
builder.append("Usage: routeradmin\n");
|
||||||
for (Map.Entry<String, UsageInfo> cmdEntry : ADMIN_USAGE.entrySet()) {
|
for (String cmdKey : ADMIN_USAGE.keySet()) {
|
||||||
UsageInfo usageInfo = cmdEntry.getValue();
|
buildHelpMsg(cmdKey, builder);
|
||||||
builder.append(" ")
|
builder.append("\n");
|
||||||
.append(cmdEntry.getKey())
|
|
||||||
.append(" ")
|
|
||||||
.append(usageInfo.args)
|
|
||||||
.append("\n");
|
|
||||||
}
|
}
|
||||||
builder.append(" -help [cmd]\n");
|
builder.append(" -help [cmd]\n");
|
||||||
}
|
}
|
||||||
@ -249,7 +348,7 @@ private static void buildUsageMsg(StringBuilder builder) {
|
|||||||
private static void printUsage(String cmd) {
|
private static void printUsage(String cmd) {
|
||||||
StringBuilder usageBuilder = new StringBuilder();
|
StringBuilder usageBuilder = new StringBuilder();
|
||||||
if (ADMIN_USAGE.containsKey(cmd)) {
|
if (ADMIN_USAGE.containsKey(cmd)) {
|
||||||
buildIndividualUsageMsg(cmd, usageBuilder);
|
buildHelpMsg(cmd, usageBuilder);
|
||||||
} else {
|
} else {
|
||||||
buildUsageMsg(usageBuilder);
|
buildUsageMsg(usageBuilder);
|
||||||
}
|
}
|
||||||
@ -353,7 +452,7 @@ private int handlePolicy(String[] args)
|
|||||||
saveOpt.setOptionalArg(true);
|
saveOpt.setOptionalArg(true);
|
||||||
Option batchSaveOpt = new Option(OPTION_BATCH_S, OPTION_BATCH_SAVE, false,
|
Option batchSaveOpt = new Option(OPTION_BATCH_S, OPTION_BATCH_SAVE, false,
|
||||||
"We will save queue policies in bulk, " +
|
"We will save queue policies in bulk, " +
|
||||||
"where users can provide XML or JSON files containing the policies. " +
|
"where users can provide XML files containing the policies. " +
|
||||||
"This command will parse the file contents and store the results " +
|
"This command will parse the file contents and store the results " +
|
||||||
"in the FederationStateStore.");
|
"in the FederationStateStore.");
|
||||||
Option formatOpt = new Option(null, "format", true,
|
Option formatOpt = new Option(null, "format", true,
|
||||||
@ -748,8 +847,59 @@ public int run(String[] args) throws Exception {
|
|||||||
return EXIT_SUCCESS;
|
return EXIT_SUCCESS;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static UsageInfo getPolicyBatchSaveUsage() {
|
||||||
|
return POLICY_BATCH_SAVE_USAGE;
|
||||||
|
}
|
||||||
|
|
||||||
|
static class RouterCmdUsageInfos {
|
||||||
|
private List<UsageInfo> usageInfos;
|
||||||
|
private List<String> helpInfos;
|
||||||
|
private Map<String, List<String>> examples;
|
||||||
|
protected Map<String, List<String>> exampleDescs;
|
||||||
|
|
||||||
|
RouterCmdUsageInfos() {
|
||||||
|
this.usageInfos = new ArrayList<>();
|
||||||
|
this.helpInfos = new ArrayList<>();
|
||||||
|
this.examples = new LinkedHashMap<>();
|
||||||
|
this.exampleDescs = new LinkedHashMap<>();
|
||||||
|
}
|
||||||
|
|
||||||
|
public RouterCmdUsageInfos addUsageInfo(UsageInfo usageInfo) {
|
||||||
|
this.usageInfos.add(usageInfo);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public RouterCmdUsageInfos addHelpInfo(String helpInfo) {
|
||||||
|
this.helpInfos.add(helpInfo);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
private RouterCmdUsageInfos addExample(String cmd, String example) {
|
||||||
|
List<String> exampleList = this.examples.getOrDefault(cmd, new ArrayList<>());
|
||||||
|
exampleList.add(example);
|
||||||
|
this.examples.put(cmd, exampleList);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
private RouterCmdUsageInfos addExampleDescs(String cmd, String exampleDesc) {
|
||||||
|
List<String> exampleDescList = this.exampleDescs.getOrDefault(cmd, new ArrayList<>());
|
||||||
|
exampleDescList.add(exampleDesc);
|
||||||
|
this.exampleDescs.put(cmd, exampleDescList);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Map<String, List<String>> getExamples() {
|
||||||
|
return examples;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public static void main(String[] args) throws Exception {
|
public static void main(String[] args) throws Exception {
|
||||||
int result = ToolRunner.run(new RouterCLI(), args);
|
int result = ToolRunner.run(new RouterCLI(), args);
|
||||||
System.exit(result);
|
System.exit(result);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@VisibleForTesting
|
||||||
|
public Map<String, RouterCmdUsageInfos> getAdminUsage(){
|
||||||
|
return ADMIN_USAGE;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -40,6 +40,7 @@
|
|||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Date;
|
import java.util.Date;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
import static org.junit.Assert.assertEquals;
|
import static org.junit.Assert.assertEquals;
|
||||||
import static org.junit.Assert.assertNotNull;
|
import static org.junit.Assert.assertNotNull;
|
||||||
@ -266,4 +267,29 @@ public void testListPolicies() throws Exception {
|
|||||||
String[] args = {"-policy", "-l", "--queue", "root.a"};
|
String[] args = {"-policy", "-l", "--queue", "root.a"};
|
||||||
assertEquals(0, rmAdminCLI.run(args));
|
assertEquals(0, rmAdminCLI.run(args));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testBuildHelpMsg() throws Exception {
|
||||||
|
Map<String, RouterCLI.RouterCmdUsageInfos> adminUsage = rmAdminCLI.getAdminUsage();
|
||||||
|
assertEquals(2, adminUsage.size());
|
||||||
|
|
||||||
|
RouterCLI.RouterCmdUsageInfos deregisterSubClusterUsageInfos =
|
||||||
|
adminUsage.get("-deregisterSubCluster");
|
||||||
|
assertNotNull(deregisterSubClusterUsageInfos);
|
||||||
|
Map<String, List<String>> dsExamplesMap = deregisterSubClusterUsageInfos.getExamples();
|
||||||
|
assertNotNull(dsExamplesMap);
|
||||||
|
assertEquals(1, dsExamplesMap.size());
|
||||||
|
List<String> dsExamples = dsExamplesMap.get("-deregisterSubCluster");
|
||||||
|
assertNotNull(dsExamples);
|
||||||
|
assertEquals(2, dsExamples.size());
|
||||||
|
|
||||||
|
RouterCLI.RouterCmdUsageInfos policyUsageInfos = adminUsage.get("-policy");
|
||||||
|
assertNotNull(policyUsageInfos);
|
||||||
|
Map<String, List<String>> policyExamplesMap = policyUsageInfos.getExamples();
|
||||||
|
assertNotNull(policyExamplesMap);
|
||||||
|
assertEquals(3, policyExamplesMap.size());
|
||||||
|
policyExamplesMap.forEach((cmd, cmdExamples) -> {
|
||||||
|
assertEquals(2, cmdExamples.size());
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -465,9 +465,165 @@ If we want to use JCache, we can configure `yarn.federation.cache.class` to `org
|
|||||||
This is a Cache implemented based on the Guava framework.
|
This is a Cache implemented based on the Guava framework.
|
||||||
If we want to use it, we can configure `yarn.federation.cache.class` to `org.apache.hadoop.yarn.server.federation.cache.FederationGuavaCache`.
|
If we want to use it, we can configure `yarn.federation.cache.class` to `org.apache.hadoop.yarn.server.federation.cache.FederationGuavaCache`.
|
||||||
|
|
||||||
|
Router command line:
|
||||||
|
|
||||||
|
- deregisterSubCluster
|
||||||
|
|
||||||
|
This command is used to `deregister subCluster`, If the interval between the heartbeat time of the subCluster, and the current time exceeds the timeout period, set the state of the subCluster to `SC_LOST`.
|
||||||
|
|
||||||
|
Uasge:
|
||||||
|
|
||||||
|
`yarn routeradmin -deregisterSubCluster [-sc|--subClusterId <subCluster Id>]`
|
||||||
|
|
||||||
|
Options:
|
||||||
|
|
||||||
|
| Property | Description |
|
||||||
|
|:--------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||||
|
| `-sc, --subClusterId [subCluster Id]` | `'-sc' option allows you to specify the sub-cluster to operate on, while the '--subClusterId' option is the long format of -sc and serves the same purpose.` |
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
If we want to deregisterSubCluster `SC-1`
|
||||||
|
|
||||||
|
- yarn routeradmin -deregisterSubCluster -sc SC-1
|
||||||
|
- yarn routeradmin -deregisterSubCluster --subClusterId SC-1
|
||||||
|
|
||||||
|
- policy
|
||||||
|
|
||||||
|
We provide a set of commands for Policy Include list policies, save policies, batch save policies.
|
||||||
|
|
||||||
|
Uasge:
|
||||||
|
|
||||||
|
`yarn routeradmin -policy -s|--save (queue;router weight;amrm weight;headroomalpha)`
|
||||||
|
|
||||||
|
`yarn routeradmin -policy -bs|--batch-save (--format xml) (-f|--input-file fileName)`
|
||||||
|
|
||||||
|
`yarn routeradmin -policy -l|--list ([--pageSize][--currentPage][--queue][--queues])`
|
||||||
|
|
||||||
|
- -s|--save (<queue;router weight;amrm weight;headroomalpha>)
|
||||||
|
|
||||||
|
This command is used to save the policy information of the queue, including queue and weight information.
|
||||||
|
|
||||||
|
How to configure `queue;router weight;amrm weight;headroomalpha`
|
||||||
|
|
||||||
|
the sum of weights for all sub-clusters in routerWeight/amrmWeight should be 1.
|
||||||
|
|
||||||
|
| Property | Description |
|
||||||
|
|:----------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||||
|
| `queue` | `Scheduled queue` |
|
||||||
|
| `router weight` | `Weight for routing applications to different subclusters.` |
|
||||||
|
| `amrm weight` | `Weight for resource request from ApplicationMaster (AM) to different subclusters' Resource Manager (RM).` |
|
||||||
|
| `headroomalpha` | `Used by policies that balance weight-based and load-based considerations in their decisions. It is recommended to use 1.0 because the load-base function is not yet complete.` |
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
We have two sub-clusters, `SC-1` and `SC-2`. We want to configure a weight policy for the `root.a` queue. The Router Weight is set to `SC-1` with a weight of `0.7` and `SC-2` with a weight of `0.3`.
|
||||||
|
The AMRM Weight is set `SC-1` to `0.6` and `SC-2` to `0.4`. We are using the default value of `0.1` for `headroomalpha`.
|
||||||
|
|
||||||
|
yarn routeradmin -policy --save root.a;SC-1:0.7,SC-2:0.3;SC-1:0.6,SC-2:0.4;1.0
|
||||||
|
|
||||||
|
yarn routeradmin -policy -s root.a;SC-1:0.7,SC-2:0.3;SC-1:0.6,SC-2:0.4;1.0
|
||||||
|
|
||||||
|
- -bs|--batch-save (--format xml) (-f|--input-file fileName)
|
||||||
|
|
||||||
|
This command can batch load weight information for queues based on the provided `federation-weights.xml` file.
|
||||||
|
|
||||||
|
| Property | Description |
|
||||||
|
|:--------------------------|:----------------------------------------------------------------------------------------------|
|
||||||
|
| `--format [xml]` | `Configuration file format, we currently only support xml format` |
|
||||||
|
| `-f, --input-file [path]` | `The path to the configuration file. Please use the absolute path of the configuration file.` |
|
||||||
|
|
||||||
|
How to configure `federation-weights.xml`
|
||||||
|
```xml
|
||||||
|
<federationWeights>
|
||||||
|
<weight>
|
||||||
|
<queue>
|
||||||
|
<name>root.a</name>
|
||||||
|
<amrmPolicyWeights>
|
||||||
|
<subClusterIdInfo>
|
||||||
|
<id>SC-1</id>
|
||||||
|
<weight>0.7</weight>
|
||||||
|
</subClusterIdInfo>
|
||||||
|
<subClusterIdInfo>
|
||||||
|
<id>SC-2</id>
|
||||||
|
<weight>0.3</weight>
|
||||||
|
</subClusterIdInfo>
|
||||||
|
</amrmPolicyWeights>
|
||||||
|
<routerPolicyWeights>
|
||||||
|
<subClusterIdInfo>
|
||||||
|
<id>SC-1</id>
|
||||||
|
<weight>0.6</weight>
|
||||||
|
</subClusterIdInfo>
|
||||||
|
<subClusterIdInfo>
|
||||||
|
<id>SC-2</id>
|
||||||
|
<weight>0.4</weight>
|
||||||
|
</subClusterIdInfo>
|
||||||
|
</routerPolicyWeights>
|
||||||
|
<headroomAlpha>1.0</headroomAlpha>
|
||||||
|
</queue>
|
||||||
|
</weight>
|
||||||
|
<weight>
|
||||||
|
<queue>
|
||||||
|
<name>root.b</name>
|
||||||
|
<amrmPolicyWeights>
|
||||||
|
<subClusterIdInfo>
|
||||||
|
<id>SC-1</id>
|
||||||
|
<weight>0.8</weight>
|
||||||
|
</subClusterIdInfo>
|
||||||
|
<subClusterIdInfo>
|
||||||
|
<id>SC-2</id>
|
||||||
|
<weight>0.2</weight>
|
||||||
|
</subClusterIdInfo>
|
||||||
|
</amrmPolicyWeights>
|
||||||
|
<routerPolicyWeights>
|
||||||
|
<subClusterIdInfo>
|
||||||
|
<id>SC-1</id>
|
||||||
|
<weight>0.6</weight>
|
||||||
|
</subClusterIdInfo>
|
||||||
|
<subClusterIdInfo>
|
||||||
|
<id>SC-2</id>
|
||||||
|
<weight>0.4</weight>
|
||||||
|
</subClusterIdInfo>
|
||||||
|
</routerPolicyWeights>
|
||||||
|
<headroomAlpha>1.0</headroomAlpha>
|
||||||
|
</queue>
|
||||||
|
</weight>
|
||||||
|
</federationWeights>
|
||||||
|
```
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
We have two sub-clusters, `SC-1` and `SC-2`. We would like to configure weights for `root.a` and `root.b` queues. We can set the weights for `root.a` and `root.b` in the `federation-weights.xml` file.
|
||||||
|
and then use the batch-save command to save the configurations in bulk.
|
||||||
|
|
||||||
|
The file name can be any file name, but it is recommended to use `federation-weights.xml`
|
||||||
|
|
||||||
|
yarn routeradmin -policy -bs --format xml -f /path/federation-weights.xml
|
||||||
|
|
||||||
|
yarn routeradmin -policy --batch-save --format xml -f /path/federation-weights.xml
|
||||||
|
|
||||||
|
- -l|--list (--pageSize --currentPage --queue --queues)
|
||||||
|
|
||||||
|
This command is used to display the configured queue weight information.
|
||||||
|
|
||||||
|
| Property | Description |
|
||||||
|
|:----------------|:-------------------------------------------------------------|
|
||||||
|
| `--pageSize` | `The number of policies displayed per page.` |
|
||||||
|
| `--currentPage` | `This parameter represents the page number to be displayed.` |
|
||||||
|
| `--queue` | `the queue we need to filter. example: root.a` |
|
||||||
|
| `--queues` | `list of queues to filter. example: root.a,root.b,root.c` |
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
We can display the list of already configured queue weight information. We can use the `--queue` option to query the weight information for a specific queue or use the `--queues` option to query the weight information for multiple queues.
|
||||||
|
|
||||||
|
yarn routeradmin -policy -l --pageSize 20 --currentPage 1 --queue root.a
|
||||||
|
|
||||||
|
yarn routeradmin -policy -list --pageSize 20 --currentPage 1 --queues root.a,root.b
|
||||||
|
|
||||||
### ON GPG:
|
### ON GPG:
|
||||||
|
|
||||||
GlobalPolicyGenerator, abbreviated as "GPG," is used for the automatic generation of global policies for subClusters.
|
GlobalPolicyGenerator, abbreviated as "GPG", is used for the automatic generation of global policies for subClusters.
|
||||||
|
|
||||||
These are extra configurations that should appear in the **conf/yarn-site.xml** for GPG. We allow only one GPG.
|
These are extra configurations that should appear in the **conf/yarn-site.xml** for GPG. We allow only one GPG.
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user