From a6a9fe17e030a809192ffaad9bd42833d6744a24 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Tue, 7 Feb 2023 03:47:07 +0800 Subject: [PATCH] YARN-3657. Federation maintenance mechanisms (simple CLI and command propagation). (#5348) --- hadoop-yarn-project/hadoop-yarn/bin/yarn | 4 + hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 6 ++ .../hadoop/yarn/client/cli/RouterCLI.java | 93 +++++++++++++++++++ .../hadoop/yarn/client/cli/TestRouterCLI.java | 64 +++++++++++++ 4 files changed, 167 insertions(+) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index f305c2744e..7fff145c03 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -50,6 +50,7 @@ function hadoop_usage hadoop_add_subcommand "fs2cs" client "converts Fair Scheduler configuration to Capacity Scheduler (EXPERIMENTAL)" hadoop_add_subcommand "rmadmin" admin "admin tools" hadoop_add_subcommand "router" daemon "run the Router daemon" + hadoop_add_subcommand "routeradmin" admin "router admin tools" hadoop_add_subcommand "schedulerconf" client "Updates scheduler configuration" hadoop_add_subcommand "scmadmin" admin "SharedCacheManager admin tools" hadoop_add_subcommand "sharedcachemanager" daemon "run the SharedCacheManager daemon" @@ -180,6 +181,9 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}" HADOOP_HEAPSIZE_MAX="${YARN_ROUTER_HEAPSIZE}" fi ;; + routeradmin) + HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.cli.RouterCLI' + ;; schedulerconf) HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.cli.SchedConfCLI' ;; diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd index ed0294c2ed..03c66b09ed 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd @@ -265,6 +265,11 @@ goto :eof ) goto :eof +:routeradmin + set CLASS=org.apache.hadoop.yarn.client.cli.RouterCLI + set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS% + goto :eof + :nodemanager set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\nm-config\log4j.properties set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\* @@ -342,6 +347,7 @@ goto :eof @echo resourcemanager run the ResourceManager @echo nodemanager run a nodemanager on each slave @echo router run the Router daemon + @echo routeradmin router admin tools @echo timelineserver run the timeline server @echo timelinereader run the timeline reader server @echo rmadmin admin tools diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java new file mode 100644 index 0000000000..3aae20ccf9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RouterCLI.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.client.cli; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.yarn.client.ClientRMProxy; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol; + +import java.io.IOException; + +public class RouterCLI extends Configured implements Tool { + + public RouterCLI() { + super(); + } + + public RouterCLI(Configuration conf) { + super(conf); + } + + private static void printHelp() { + StringBuilder summary = new StringBuilder(); + summary.append("router-admin is the command to execute " + + "YARN Federation administrative commands.\n"); + StringBuilder helpBuilder = new StringBuilder(); + System.out.println(summary); + helpBuilder.append(" -help [cmd]: Displays help for the given command or all commands" + + " if none is specified."); + System.out.println(helpBuilder); + System.out.println(); + ToolRunner.printGenericCommandUsage(System.out); + } + + protected ResourceManagerAdministrationProtocol createAdminProtocol() + throws IOException { + // Get the current configuration + final YarnConfiguration conf = new YarnConfiguration(getConf()); + return ClientRMProxy.createRMProxy(conf, ResourceManagerAdministrationProtocol.class); + } + + private static void buildUsageMsg(StringBuilder builder) { + builder.append("router-admin is only used in Yarn Federation Mode.\n"); + builder.append("Usage: router-admin\n"); + builder.append(" -help" + " [cmd]\n"); + } + + private static void printUsage() { + StringBuilder usageBuilder = new StringBuilder(); + buildUsageMsg(usageBuilder); + System.err.println(usageBuilder); + ToolRunner.printGenericCommandUsage(System.err); + } + + @Override + public int run(String[] args) throws Exception { + YarnConfiguration yarnConf = getConf() == null ? + new YarnConfiguration() : new YarnConfiguration(getConf()); + boolean isFederationEnabled = yarnConf.getBoolean(YarnConfiguration.FEDERATION_ENABLED, + YarnConfiguration.DEFAULT_FEDERATION_ENABLED); + + if (args.length < 1 || !isFederationEnabled) { + printUsage(); + return -1; + } + + String cmd = args[0]; + if ("-help".equals(cmd)) { + printHelp(); + return 0; + } + + return 0; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java new file mode 100644 index 0000000000..a8401f8870 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRouterCLI.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.client.cli; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; + +public class TestRouterCLI { + + private ResourceManagerAdministrationProtocol admin; + private RouterCLI rmAdminCLI; + + @Before + public void setup() throws Exception { + + admin = mock(ResourceManagerAdministrationProtocol.class); + Configuration config = new Configuration(); + config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true); + + rmAdminCLI = new RouterCLI(config) { + @Override + protected ResourceManagerAdministrationProtocol createAdminProtocol() { + return admin; + } + }; + } + + @Test + public void testHelp() throws Exception { + PrintStream oldOutPrintStream = System.out; + PrintStream oldErrPrintStream = System.err; + ByteArrayOutputStream dataOut = new ByteArrayOutputStream(); + ByteArrayOutputStream dataErr = new ByteArrayOutputStream(); + System.setOut(new PrintStream(dataOut)); + System.setErr(new PrintStream(dataErr)); + + String[] args = {"-help"}; + assertEquals(0, rmAdminCLI.run(args)); + } +}