big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-7384. Remove apiserver cmd and merge service cmd into application cmd. Contributed by Billie Rinaldi

Browse Source
This commit is contained in:
Jian He 2017-10-28 23:04:19 -07:00
parent 8ec9e72cc3
commit ba7ed7b66a
54 changed files with 842 additions and 3247 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
NOTICE.txt
View File

@ -582,10 +582,6 @@ Ehcache 3.3.1,
which has the following notices:
* Ehcache V3 Copyright 2014-2016 Terracotta, Inc.
JCommander (https://github.com/cbeust/jcommander),
which has the following notices:
* Copyright 2010 Cedric Beust cedric@beust.com
The binary distribution of this product bundles binaries of
snakeyaml (https://bitbucket.org/asomov/snakeyaml),
which has the following notices:

6
hadoop-project/pom.xml
View File

@ -146,7 +146,6 @@
<swagger-annotations-version>1.5.4</swagger-annotations-version>
<snakeyaml.version>1.16</snakeyaml.version>
<jcommander.version>1.48</jcommander.version>
</properties>
<dependencyManagement>
@ -1353,11 +1352,6 @@
<artifactId>snakeyaml</artifactId>
<version>${snakeyaml.version}</version>
</dependency>
<dependency>
<groupId>com.beust</groupId>
<artifactId>jcommander</artifactId>
<version>${jcommander.version}</version>
</dependency>
</dependencies>
</dependencyManagement>

27
hadoop-yarn-project/hadoop-yarn/bin/yarn
View File

@ -31,8 +31,7 @@ function hadoop_usage
hadoop_add_option "--hosts filename" "list of hosts to use in worker mode"
hadoop_add_option "--workers" "turn on worker mode"
hadoop_add_subcommand "apiserver" daemon "run the api-server for deploying/managing services on YARN"
hadoop_add_subcommand "application" client "prints application(s) report/kill application"
hadoop_add_subcommand "app|application" client "prints application(s) report/kill application/manage long running application"
hadoop_add_subcommand "applicationattempt" client "prints applicationattempt(s) report"
hadoop_add_subcommand "classpath" client "prints the class path needed to get the hadoop jar and the required libraries"
hadoop_add_subcommand "cluster" client "prints cluster information"
@ -51,7 +50,6 @@ function hadoop_usage
hadoop_add_subcommand "router" daemon "run the Router daemon"
hadoop_add_subcommand "schedulerconf" client "Updates scheduler configuration"
hadoop_add_subcommand "scmadmin" admin "SharedCacheManager admin tools"
hadoop_add_subcommand "service" client "deploy/manage a service on YARN"
hadoop_add_subcommand "sharedcachemanager" daemon "run the SharedCacheManager daemon"
hadoop_add_subcommand "timelinereader" client "run the timeline reader server"
hadoop_add_subcommand "timelineserver" daemon "run the timeline server"
@ -71,9 +69,10 @@ function yarncmd_case
shift
case ${subcmd} in
apiserver)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.yarn.service.webapp.ApiServerWebApp'
app|application|applicationattempt|container)
HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.ApplicationCLI
set -- "${subcmd}" "$@"
HADOOP_SUBCMD_ARGS=("$@")
local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\
${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
${HADOOP_HDFS_HOME}/${HDFS_DIR},\
@ -83,11 +82,6 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
hadoop_translate_cygwin_path sld
hadoop_add_param HADOOP_OPTS service.libdir "-Dservice.libdir=${sld}"
;;
application|applicationattempt|container)
HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.ApplicationCLI
set -- "${subcmd}" "$@"
HADOOP_SUBCMD_ARGS=("$@")
;;
classpath)
hadoop_do_classpath_subcommand HADOOP_CLASSNAME "$@"
;;
@ -169,17 +163,6 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
scmadmin)
HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.SCMAdmin'
;;
service)
HADOOP_CLASSNAME='org.apache.hadoop.yarn.service.client.ServiceCLI'
local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\
${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
${HADOOP_HDFS_HOME}/${HDFS_DIR},\
${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
hadoop_translate_cygwin_path sld
hadoop_add_param HADOOP_OPTS service.libdir "-Dservice.libdir=${sld}"
;;
sharedcachemanager)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager'

18
hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
View File

@ -149,18 +149,6 @@
#
#export YARN_ROUTER_OPTS=
###
# Services API specific parameters
###
# Specify the JVM options to be used when starting the services API.
#
# These options will be appended to the options specified as HADOOP_OPTS
# and therefore may override any similar flags set in HADOOP_OPTS
#
# See ResourceManager for some examples
#
#export YARN_APISERVER_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-apiserver.log-$(date +'%Y%m%d%H%M')"
###
# Registry DNS specific parameters
###
@ -172,3 +160,9 @@
# By default, Hadoop uses jsvc which needs to know to launch a
# server jvm.
# export YARN_REGISTRYDNS_SECURE_EXTRA_OPTS="-jvm server"
###
# YARN Services parameters
###
# Directory containing service examples
# export YARN_SERVICE_EXAMPLES_DIR = $HADOOP_YARN_HOME/share/hadoop/yarn/yarn-service-examples

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
View File

@ -101,11 +101,6 @@
</plugins>
</build>
<dependencies>
<dependency>
<groupId>com.beust</groupId>
<artifactId>jcommander</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
View File

@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.service;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Options;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
@ -34,7 +36,6 @@
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
import org.apache.hadoop.yarn.service.client.params.ServiceAMArgs;
import org.apache.hadoop.yarn.service.monitor.ServiceMonitor;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
@ -51,7 +52,9 @@ public class ServiceMaster extends CompositeService {
private static final Logger LOG =
LoggerFactory.getLogger(ServiceMaster.class);
private static ServiceAMArgs amArgs;
public static final String YARNFILE_OPTION = "yarnfile";
private static String serviceDefPath;
protected ServiceContext context;
public ServiceMaster(String name) {
@ -108,7 +111,7 @@ protected ContainerId getAMContainerId() throws BadClusterStateException {
}
protected Path getAppDir() {
return new Path(amArgs.getServiceDefPath()).getParent();
return new Path(serviceDefPath).getParent();
}
protected ServiceScheduler createServiceScheduler(ServiceContext context)
@ -119,7 +122,7 @@ protected ServiceScheduler createServiceScheduler(ServiceContext context)
protected void loadApplicationJson(ServiceContext context,
SliderFileSystem fs) throws IOException {
context.service = ServiceApiUtil
.loadServiceFrom(fs, new Path(amArgs.getServiceDefPath()));
.loadServiceFrom(fs, new Path(serviceDefPath));
LOG.info(context.service.toString());
}
@ -138,14 +141,18 @@ private void printSystemEnv() {
public static void main(String[] args) throws Exception {
Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
StringUtils.startupShutdownMessage(ServiceMaster.class, args, LOG);
amArgs = new ServiceAMArgs(args);
amArgs.parse();
try {
ServiceMaster serviceMaster = new ServiceMaster("Service Master");
ShutdownHookManager.get()
.addShutdownHook(new CompositeServiceShutdownHook(serviceMaster), 30);
YarnConfiguration conf = new YarnConfiguration();
new GenericOptionsParser(conf, args);
Options opts = new Options();
opts.addOption(YARNFILE_OPTION, true, "HDFS path to JSON service " +
"specification");
opts.getOption(YARNFILE_OPTION).setRequired(true);
GenericOptionsParser parser = new GenericOptionsParser(conf, opts, args);
CommandLine cmdLine = parser.getCommandLine();
serviceMaster.serviceDefPath = cmdLine.getOptionValue(YARNFILE_OPTION);
serviceMaster.init(conf);
serviceMaster.start();
} catch (Throwable t) {

113
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceCLI.java
View File

@ -1,113 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client;
import com.beust.jcommander.ParameterException;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.client.params.ClientArgs;
import org.apache.hadoop.yarn.service.exceptions.SliderException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.FileNotFoundException;
import static org.apache.hadoop.yarn.service.client.params.SliderActions.*;
public class ServiceCLI {
private static final Logger LOG =
LoggerFactory.getLogger(ServiceClient.class);
protected ServiceClient client;
int exec(ClientArgs args) throws Throwable {
if (StringUtils.isEmpty(args.getAction())) {
System.out.println(args.usage());
return -1;
}
switch (args.getAction()) {
case ACTION_BUILD: // Upload app json onto hdfs
client.actionBuild(args.getActionBuildArgs());
break;
case ACTION_START: // start the app with the pre-uploaded app json on hdfs
client.actionStart(args.getClusterName());
break;
case ACTION_CREATE: // create == build + start
client.actionCreate(args.getActionCreateArgs());
break;
case ACTION_STATUS:
Service app = client.getStatus(args.getClusterName());
System.out.println(app);
break;
case ACTION_FLEX:
try {
client.actionFlexByCLI(args);
} catch (FileNotFoundException e) {
System.err.println(
args.getClusterName() + " doesn't exist: " + e.getMessage());
return -1;
}
break;
case ACTION_STOP:
client.actionStop(args.getClusterName(), false);
break;
case ACTION_DESTROY: // Destroy can happen only if app is already stopped
client.actionDestroy(args.getClusterName());
break;
case ACTION_DEPENDENCY: // upload dependency jars
client.actionDependency(args.getActionDependencyArgs());
break;
case ACTION_UPDATE:
client.updateLifetime(args.getClusterName(),
args.getActionUpdateArgs().lifetime);
break;
case ACTION_HELP:
LOG.info(args.usage());
break;
default:
LOG.info("NOT IMPLEMENTED: " + args.getAction());
LOG.info(args.usage());
return -1;
}
return 0;
}
public ServiceCLI() {
createServiceClient();
}
protected void createServiceClient() {
client = new ServiceClient();
client.init(new YarnConfiguration());
client.start();
}
public static void main(String[] args) throws Throwable {
ClientArgs clientArgs = new ClientArgs(args);
try {
clientArgs.parse();
} catch (ParameterException | SliderException e) {
System.err.println(e.getMessage());
System.exit(-1);
}
ServiceCLI cli = new ServiceCLI();
int res = cli.exec(clientArgs);
System.exit(res);
}
}

187
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
View File

@ -34,7 +34,6 @@
import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
import org.apache.hadoop.registry.client.binding.RegistryUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.util.VersionInfo;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
@ -50,6 +49,7 @@
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.AppAdminClient;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@ -65,13 +65,6 @@
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.api.records.ServiceState;
import org.apache.hadoop.yarn.service.client.params.AbstractClusterBuildingActionArgs;
import org.apache.hadoop.yarn.service.client.params.ActionCreateArgs;
import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs;
import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs;
import org.apache.hadoop.yarn.service.client.params.Arguments;
import org.apache.hadoop.yarn.service.client.params.ClientArgs;
import org.apache.hadoop.yarn.service.client.params.CommonArgs;
import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
@ -80,7 +73,6 @@
import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
import org.apache.hadoop.yarn.service.exceptions.SliderException;
import org.apache.hadoop.yarn.service.exceptions.UsageException;
import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
import org.apache.hadoop.yarn.service.provider.ProviderUtils;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
@ -107,16 +99,14 @@
import java.util.concurrent.ConcurrentHashMap;
import static org.apache.hadoop.yarn.api.records.YarnApplicationState.*;
import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_CREATE;
import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_FLEX;
import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_QUEUE;
import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.jsonSerDeser;
import static org.apache.hadoop.yarn.service.utils.SliderUtils.*;
@InterfaceAudience.Public
@InterfaceStability.Unstable
public class ServiceClient extends CompositeService
implements SliderExitCodes, YarnServiceConstants {
public class ServiceClient extends AppAdminClient implements SliderExitCodes,
YarnServiceConstants {
private static final Logger LOG =
LoggerFactory.getLogger(ServiceClient.class);
private SliderFileSystem fs;
@ -136,10 +126,6 @@ public class ServiceClient extends CompositeService
private static EnumSet<YarnApplicationState> preRunningStates =
EnumSet.of(NEW, NEW_SAVING, SUBMITTED, ACCEPTED);
public ServiceClient() {
super(ServiceClient.class.getName());
}
@Override protected void serviceInit(Configuration configuration)
throws Exception {
fs = new SliderFileSystem(configuration);
@ -157,28 +143,60 @@ protected void serviceStop() throws Exception {
super.serviceStop();
}
private Service loadAppJsonFromLocalFS(
AbstractClusterBuildingActionArgs args) throws IOException {
File file = args.getFile();
public Service loadAppJsonFromLocalFS(String fileName, String serviceName,
Long lifetime, String queue) throws IOException, YarnException {
File file = new File(fileName);
if (!file.exists() && fileName.equals(file.getName())) {
String examplesDirStr = System.getenv("YARN_SERVICE_EXAMPLES_DIR");
String[] examplesDirs;
if (examplesDirStr == null) {
String yarnHome = System
.getenv(ApplicationConstants.Environment.HADOOP_YARN_HOME.key());
examplesDirs = new String[]{
yarnHome + "/share/hadoop/yarn/yarn-service-examples",
yarnHome + "/yarn-service-examples"
};
} else {
examplesDirs = StringUtils.split(examplesDirStr, ":");
}
for (String dir : examplesDirs) {
file = new File(MessageFormat.format("{0}/{1}/{2}.json",
dir, fileName, fileName));
if (file.exists()) {
break;
}
// Then look for secondary location.
file = new File(MessageFormat.format("{0}/{1}.json",
dir, fileName));
if (file.exists()) {
break;
}
}
}
if (!file.exists()) {
throw new YarnException("File or example could not be found: " +
fileName);
}
Path filePath = new Path(file.getAbsolutePath());
LOG.info("Loading service definition from: " + filePath);
Service service = jsonSerDeser
.load(FileSystem.getLocal(getConfig()), filePath);
if (args.lifetime > 0) {
service.setLifetime(args.lifetime);
if (!StringUtils.isEmpty(serviceName)) {
service.setName(serviceName);
}
if (!StringUtils.isEmpty(args.getServiceName())) {
service.setName(args.getServiceName());
if (lifetime != null && lifetime > 0) {
service.setLifetime(lifetime);
}
if (!StringUtils.isEmpty(args.queue)) {
service.setQueue(args.queue);
if (!StringUtils.isEmpty(queue)) {
service.setQueue(queue);
}
return service;
}
public int actionBuild(AbstractClusterBuildingActionArgs args)
throws IOException, YarnException {
return actionBuild(loadAppJsonFromLocalFS(args));
public int actionSave(String fileName, String serviceName, Long lifetime,
String queue) throws IOException, YarnException {
return actionBuild(loadAppJsonFromLocalFS(fileName, serviceName,
lifetime, queue));
}
public int actionBuild(Service service)
@ -189,43 +207,13 @@ public int actionBuild(Service service)
return EXIT_SUCCESS;
}
public int actionCreate(ActionCreateArgs args)
throws IOException, YarnException {
Service serviceDef;
if (args.file != null) {
serviceDef = loadAppJsonFromLocalFS(args);
} else if (!StringUtils.isEmpty(args.example)) {
// create an example service
args.file = findExampleService(args);
serviceDef = loadAppJsonFromLocalFS(args);
} else {
throw new YarnException("No service definition provided!");
}
actionCreate(serviceDef);
public int actionLaunch(String fileName, String serviceName, Long lifetime,
String queue) throws IOException, YarnException {
actionCreate(loadAppJsonFromLocalFS(fileName, serviceName, lifetime,
queue));
return EXIT_SUCCESS;
}
private File findExampleService(ActionCreateArgs args) throws YarnException {
String yarnHome = System
.getenv(ApplicationConstants.Environment.HADOOP_YARN_HOME.key());
// First look for the standard location.
File file = new File(MessageFormat
.format("{0}/share/hadoop/yarn/yarn-service-examples/{1}/{2}.json",
yarnHome, args.example, args.example));
if (file.exists()) {
return file;
}
// Then look for secondary location.
file = new File(MessageFormat
.format("{0}/yarn-service-examples/{1}/{2}.json", yarnHome,
args.example, args.example));
if (file.exists()) {
return file;
}
throw new YarnException(
"Example service " + args.example + " does not exist!");
}
public ApplicationId actionCreate(Service service)
throws IOException, YarnException {
String serviceName = service.getName();
@ -244,14 +232,12 @@ public ApplicationId actionCreate(Service service)
return appId;
}
// Called by ServiceCLI
protected int actionFlexByCLI(ClientArgs args)
throws YarnException, IOException {
ActionFlexArgs flexArgs = args.getActionFlexArgs();
public int actionFlex(String serviceName, Map<String, String>
componentCountStrings) throws YarnException, IOException {
Map<String, Long> componentCounts =
new HashMap<>(flexArgs.getComponentMap().size());
new HashMap<>(componentCountStrings.size());
Service persistedService =
ServiceApiUtil.loadService(fs, flexArgs.getServiceName());
ServiceApiUtil.loadService(fs, serviceName);
if (!StringUtils.isEmpty(persistedService.getId())) {
cachedAppIds.put(persistedService.getName(),
ApplicationId.fromString(persistedService.getId()));
@ -260,8 +246,7 @@ protected int actionFlexByCLI(ClientArgs args)
+ " appId is null, may be not submitted to YARN yet");
}
for (Map.Entry<String, String> entry : flexArgs.getComponentMap()
.entrySet()) {
for (Map.Entry<String, String> entry : componentCountStrings.entrySet()) {
String compName = entry.getKey();
ServiceApiUtil.validateNameFormat(compName, getConfig());
Component component = persistedService.getComponent(compName);
@ -272,11 +257,7 @@ protected int actionFlexByCLI(ClientArgs args)
parseNumberOfContainers(component, entry.getValue());
componentCounts.put(compName, numberOfContainers);
}
// throw usage exception if no changes proposed
if (componentCounts.size() == 0) {
actionHelp(ACTION_FLEX, args);
}
flexComponents(args.getClusterName(), componentCounts, persistedService);
flexComponents(serviceName, componentCounts, persistedService);
return EXIT_SUCCESS;
}
@ -372,6 +353,11 @@ private Map<String, Long> flexComponents(String serviceName,
return original;
}
public int actionStop(String serviceName)
throws YarnException, IOException {
return actionStop(serviceName, true);
}
public int actionStop(String serviceName, boolean waitForAppStopped)
throws YarnException, IOException {
ServiceApiUtil.validateNameFormat(serviceName, getConfig());
@ -442,7 +428,8 @@ public int actionStop(String serviceName, boolean waitForAppStopped)
return EXIT_SUCCESS;
}
public int actionDestroy(String serviceName) throws Exception {
public int actionDestroy(String serviceName) throws YarnException,
IOException {
ServiceApiUtil.validateNameFormat(serviceName, getConfig());
verifyNoLiveAppInRM(serviceName, "destroy");
@ -461,7 +448,11 @@ public int actionDestroy(String serviceName) throws Exception {
throw new YarnException(message);
}
}
deleteZKNode(serviceName);
try {
deleteZKNode(serviceName);
} catch (Exception e) {
throw new IOException("Could not delete zk node for " + serviceName, e);
}
String registryPath = ServiceRegistryUtils.registryPathForInstance(serviceName);
try {
getRegistryClient().delete(registryPath, true);
@ -517,11 +508,6 @@ private synchronized CuratorFramework getCuratorClient()
return curatorClient;
}
private int actionHelp(String actionName, CommonArgs args)
throws YarnException, IOException {
throw new UsageException(CommonArgs.usage(args, actionName));
}
private void verifyNoLiveAppInRM(String serviceName, String action)
throws IOException, YarnException {
Set<String> types = new HashSet<>(1);
@ -634,9 +620,9 @@ private String buildCommandLine(String serviceName, Configuration conf,
CLI.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR);
}
CLI.add(ServiceMaster.class.getCanonicalName());
CLI.add(ACTION_CREATE, serviceName);
//TODO debugAM CLI.add(Arguments.ARG_DEBUG)
CLI.add(Arguments.ARG_SERVICE_DEF_PATH, new Path(appRootDir, serviceName + ".json"));
CLI.add("-" + ServiceMaster.YARNFILE_OPTION, new Path(appRootDir,
serviceName + ".json"));
// pass the registry binding
CLI.addConfOptionToCLI(conf, RegistryConstants.KEY_REGISTRY_ZK_ROOT,
RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT);
@ -841,6 +827,24 @@ public ServiceState convertState(FinalApplicationStatus status) {
return ServiceState.ACCEPTED;
}
public String getStatusString(String appId)
throws IOException, YarnException {
ApplicationReport appReport =
yarnClient.getApplicationReport(ApplicationId.fromString(appId));
if (appReport.getYarnApplicationState() != RUNNING) {
return "";
}
if (StringUtils.isEmpty(appReport.getHost())) {
return "";
}
ClientAMProtocol amProxy =
createAMProxy(appReport.getHost(), appReport.getRpcPort());
GetStatusResponseProto response =
amProxy.getStatus(GetStatusRequestProto.newBuilder().build());
return response.getStatus();
}
public Service getStatus(String serviceName)
throws IOException, YarnException {
ServiceApiUtil.validateNameFormat(serviceName, getConfig());
@ -877,7 +881,11 @@ public YarnClient getYarnClient() {
return this.yarnClient;
}
public int actionDependency(ActionDependencyArgs args)
public int enableFastLaunch() throws IOException, YarnException {
return actionDependency(true);
}
public int actionDependency(boolean overwrite)
throws IOException, YarnException {
String currentUser = RegistryUtils.currentUser();
LOG.info("Running command as user {}", currentUser);
@ -886,11 +894,10 @@ public int actionDependency(ActionDependencyArgs args)
// Check if dependency has already been uploaded, in which case log
// appropriately and exit success (unless overwrite has been requested)
if (fs.isFile(dependencyLibTarGzip) && !args.overwrite) {
if (fs.isFile(dependencyLibTarGzip) && !overwrite) {
System.out.println(String.format(
"Dependency libs are already uploaded to %s. Use %s "
+ "if you want to re-upload", dependencyLibTarGzip.toUri(),
Arguments.ARG_OVERWRITE));
"Dependency libs are already uploaded to %s.", dependencyLibTarGzip
.toUri()));
return EXIT_SUCCESS;
}

139
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
View File

@ -1,139 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
import org.apache.hadoop.yarn.service.exceptions.UsageException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
/**
* Base args for all actions
*/
public abstract class AbstractActionArgs extends ArgOps implements Arguments {
protected static final Logger log =
LoggerFactory.getLogger(AbstractActionArgs.class);
protected AbstractActionArgs() {
}
/**
* This is the default parameter
*/
@Parameter
public List<String> parameters = new ArrayList<>();
/**
* get the name: relies on arg 1 being the cluster name in all operations
* @return the name argument, null if there is none
*/
public String getServiceName() {
return (parameters.isEmpty()) ? null : parameters.get(0);
}
/**
-D name=value
Define an configuration option which overrides any options in
the configuration XML files of the image or in the image configuration
directory. The values will be persisted.
Configuration options are only passed to the cluster when creating or reconfiguring a cluster.
*/
@Parameter(names = ARG_DEFINE, arity = 1, description = "Definitions", hidden = true)
public List<String> definitions = new ArrayList<>();
/**
* System properties
*/
@Parameter(names = {ARG_SYSPROP}, arity = 1,
description = "system properties in the form name value" +
" These are set after the JVM is started.",
hidden = true)
public List<String> sysprops = new ArrayList<>(0);
@Parameter(names = ARG_DEBUG, description = "Debug mode", hidden = true)
public boolean debug = false;
/**
* Get the min #of params expected
* @return the min number of params in the {@link #parameters} field
*/
public int getMinParams() {
return 1;
}
/**
* Get the name of the action
* @return the action name
*/
public abstract String getActionName() ;
/**
* Get the max #of params expected
* @return the number of params in the {@link #parameters} field;
*/
public int getMaxParams() {
return getMinParams();
}
public void validate() throws BadCommandArgumentsException, UsageException {
int minArgs = getMinParams();
int actionArgSize = parameters.size();
if (minArgs > actionArgSize) {
throw new BadCommandArgumentsException(
ErrorStrings.ERROR_NOT_ENOUGH_ARGUMENTS + getActionName() +
", Expected minimum " + minArgs + " but got " + actionArgSize);
}
int maxArgs = getMaxParams();
if (maxArgs == -1) {
maxArgs = minArgs;
}
if (actionArgSize > maxArgs) {
String message = String.format("%s for action %s: limit is %d but saw %d: ",
ErrorStrings.ERROR_TOO_MANY_ARGUMENTS,
getActionName(), maxArgs,
actionArgSize);
log.error(message);
int index = 1;
StringBuilder buf = new StringBuilder(message);
for (String actionArg : parameters) {
log.error("[{}] \"{}\"", index++, actionArg);
buf.append(" \"").append(actionArg).append("\" ");
}
throw new BadCommandArgumentsException(buf.toString());
}
}
@Override
public String toString() {
return super.toString() + ": " + getActionName();
}
}

28
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractArgsDelegate.java
View File

@ -1,28 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import org.apache.hadoop.yarn.service.client.params.ArgOps;
import org.apache.hadoop.yarn.service.client.params.Arguments;
/**
* Base class for all the delegates
*/
public class AbstractArgsDelegate extends ArgOps implements Arguments {
}

46
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
View File

@ -1,46 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import java.io.File;
/**
* Abstract Action to build things; shares args across build and
* list
*/
public abstract class AbstractClusterBuildingActionArgs
extends AbstractActionArgs {
@Parameter(names = { ARG_FILE, ARG_FILE_SHORT },
description = "The path to the service definition file in JSON format.")
public File file;
public File getFile() {
return file;
}
@Parameter(names = {
ARG_QUEUE, ARG_SHORT_QUEUE}, description = "Queue to submit the service")
public String queue;
@Parameter(names = {
ARG_LIFETIME }, description = "Lifetime of the service from the time of request")
public long lifetime;
}

39
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
View File

@ -1,39 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameters;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
@Parameters(commandNames = { SliderActions.ACTION_BUILD},
commandDescription = SliderActions.DESCRIBE_ACTION_BUILD)
public class ActionBuildArgs extends AbstractClusterBuildingActionArgs {
@Override
public String getActionName() {
return SliderActions.ACTION_BUILD;
}
@Override
public void validate() throws BadCommandArgumentsException {
if (file == null) {
throw new BadCommandArgumentsException("No service definition provided.");
}
}
}

71
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionClientArgs.java
View File

@ -1,71 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
import org.apache.hadoop.yarn.service.client.params.SliderActions;
import java.io.File;
@Parameters(commandNames = { SliderActions.ACTION_CLIENT},
commandDescription = SliderActions.DESCRIBE_ACTION_CLIENT)
public class ActionClientArgs extends AbstractActionArgs {
@Override
public String getActionName() {
return SliderActions.ACTION_CLIENT;
}
@Parameter(names = {ARG_INSTALL},
description = "Install client")
public boolean install;
@Parameter(names = {ARG_NAME},
description = "The name of the service")
public String name;
@Parameter(names = {ARG_PACKAGE},
description = "Path to app package")
public String packageURI;
@Parameter(names = {ARG_DEST},
description = "The location where to install the client")
public File installLocation;
@Parameter(names = {ARG_CONFIG},
description = "Client configuration")
public File clientConfig;
/**
* Get the min #of params expected
*
* @return the min number of params in the {@link #parameters} field
*/
public int getMinParams() {
return 0;
}
@Override
public int getMaxParams() {
return 1;
}
}

46
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
View File

@ -1,46 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
@Parameters(commandNames = { SliderActions.ACTION_CREATE},
commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
public class ActionCreateArgs extends AbstractClusterBuildingActionArgs {
@Parameter(names = { ARG_EXAMPLE, ARG_EXAMPLE_SHORT },
description = "The name of the example service such as sleeper")
public String example;
@Override
public String getActionName() {
return SliderActions.ACTION_CREATE;
}
@Override
public void validate() throws BadCommandArgumentsException {
if (file == null && example == null) {
throw new BadCommandArgumentsException("No service definition provided.");
}
}
}

65
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDependencyArgs.java
View File

@ -1,65 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import org.apache.hadoop.yarn.service.exceptions.UsageException;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
@Parameters(commandNames = { SliderActions.ACTION_DEPENDENCY },
commandDescription = SliderActions.DESCRIBE_ACTION_DEPENDENCY)
public class ActionDependencyArgs extends AbstractActionArgs {
@Override
public String getActionName() {
return SliderActions.ACTION_DEPENDENCY;
}
@Parameter(names = { ARG_UPLOAD }, required = true,
description = "Upload AM libraries to HDFS for this client version")
public boolean upload;
@Parameter(names = { ARG_OVERWRITE },
description = "Overwrite current uploaded dependency libs")
public boolean overwrite = false;
/**
* Get the min #of params expected
*
* @return the min number of params in the {@link #parameters} field
*/
public int getMinParams() {
return 0;
}
@Override
public int getMaxParams() {
return 1;
}
@Override
public void validate() throws BadCommandArgumentsException, UsageException {
super.validate();
if (!upload) {
throw new UsageException("Option " + ARG_UPLOAD + " is mandatory");
}
}
}

37
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionDestroyArgs.java
View File

@ -1,37 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
@Parameters(commandNames = { SliderActions.ACTION_DESTROY},
commandDescription = SliderActions.DESCRIBE_ACTION_DESTROY)
public class ActionDestroyArgs extends AbstractActionArgs {
@Override
public String getActionName() {
return SliderActions.ACTION_DESTROY;
}
@Parameter(names = {ARG_FORCE},
description = "force the operation")
public boolean force;
}

49
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExistsArgs.java
View File

@ -1,49 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
import org.apache.hadoop.yarn.service.client.params.SliderActions;
import java.io.File;
@Parameters(commandNames = { SliderActions.ACTION_EXISTS},
commandDescription = SliderActions.DESCRIBE_ACTION_EXISTS)
public class ActionExistsArgs extends AbstractActionArgs {
@Override
public String getActionName() {
return SliderActions.ACTION_EXISTS;
}
@Parameter(names = {ARG_LIVE},
description = "verify that the service is running")
public boolean live;
@Parameter(names = {ARG_STATE},
description = "verify that the service is in the specific YARN state")
public String state = "";
@Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
description = "output file for any service report")
public File out;
}

50
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFlexArgs.java
View File

@ -1,50 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameters;
import com.beust.jcommander.ParametersDelegate;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import java.util.List;
import java.util.Map;
@Parameters(commandNames = { SliderActions.ACTION_FLEX},
commandDescription = SliderActions.DESCRIBE_ACTION_FLEX)
public class ActionFlexArgs extends AbstractActionArgs {
@Override
public String getActionName() {
return SliderActions.ACTION_FLEX;
}
@ParametersDelegate
public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate();
/**
* Get the component mapping (may be empty, but never null)
* @return mapping
* @throws BadCommandArgumentsException parse problem
*/
public Map<String, String> getComponentMap() throws
BadCommandArgumentsException {
return componentDelegate.getComponentMap();
}
}

56
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionFreezeArgs.java
View File

@ -1,56 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.beust.jcommander.ParametersDelegate;
@Parameters(commandNames = { SliderActions.ACTION_STOP },
commandDescription = SliderActions.DESCRIBE_ACTION_FREEZE)
public class ActionFreezeArgs extends AbstractActionArgs implements
WaitTimeAccessor {
@Override
public String getActionName() {
return SliderActions.ACTION_STOP;
}
public static final String FREEZE_COMMAND_ISSUED = "stop command issued";
@ParametersDelegate
public WaitArgsDelegate waitDelegate = new WaitArgsDelegate();
@Override
public int getWaittime() {
return waitDelegate.getWaittime();
}
@Override
public void setWaittime(int waittime) {
waitDelegate.setWaittime(waittime);
}
@Parameter(names={ARG_MESSAGE},
description = "reason for the operation")
public String message = FREEZE_COMMAND_ISSUED;
@Parameter(names = {ARG_FORCE},
description = "force the operation")
public boolean force;
}

44
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionHelpArgs.java
View File

@ -1,44 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameters;
import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
import org.apache.hadoop.yarn.service.client.params.SliderActions;
/**
* The Help command
*/
@Parameters(commandNames = { SliderActions.ACTION_HELP},
commandDescription = SliderActions.DESCRIBE_ACTION_HELP)
public class ActionHelpArgs extends AbstractActionArgs {
@Override
public String getActionName() {
return SliderActions.ACTION_HELP;
}
/**
* Get the min #of params expected
* @return the min number of params in the {@link #parameters} field
*/
@Override
public int getMinParams() {
return 0;
}
}

76
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionKeytabArgs.java
View File

@ -1,76 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
import org.apache.hadoop.yarn.service.client.params.SliderActions;
@Parameters(commandNames = { SliderActions.ACTION_KEYTAB},
commandDescription = SliderActions.DESCRIBE_ACTION_KEYTAB)
public class ActionKeytabArgs extends AbstractActionArgs {
public ActionKeytabArgs() {
super();
}
@Override
public String getActionName() {
return SliderActions.ACTION_INSTALL_KEYTAB;
}
@Parameter(names = {ARG_KEYTABINSTALL},
description = "Install the keytab")
public boolean install;
@Parameter(names = {ARG_KEYTABDELETE},
description = "Delete the keytab")
public boolean delete;
@Parameter(names = {ARG_KEYTABLIST},
description = "List of installed keytabs")
public boolean list;
@Parameter(names = {ARG_KEYTAB},
description = "Path or name of the keytab")
public String keytab;
@Parameter(names = {ARG_FOLDER},
description = "The name of the folder in which to store the keytab")
public String folder;
@Parameter(names = {ARG_OVERWRITE}, description = "Overwrite existing keytab")
public boolean overwrite = false;
/**
* Get the min #of params expected
* @return the min number of params in the {@link #parameters} field
*/
public int getMinParams() {
return 0;
}
@Override
public int getMaxParams() {
return 3;
}
}

76
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionListArgs.java
View File

@ -1,76 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import java.util.HashSet;
import java.util.Set;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
import org.apache.hadoop.yarn.service.client.params.SliderActions;
@Parameters(commandNames = { SliderActions.ACTION_LIST},
commandDescription = SliderActions.DESCRIBE_ACTION_LIST)
public class ActionListArgs extends AbstractActionArgs {
@Override
public String getActionName() {
return SliderActions.ACTION_LIST;
}
@Parameter(names = {ARG_LIVE},
description = "List only live service instances")
public boolean live;
@Parameter(names = {ARG_STATE},
description = "list only applications in the specific YARN state")
public String state = "";
@Parameter(names = {ARG_VERBOSE},
description = "print out information in details")
public boolean verbose = false;
@Parameter(names = {ARG_CONTAINERS},
description = "List containers of a service instance")
public boolean containers;
@Parameter(names = {ARG_VERSION},
description = "Filter containers by app version (used with " +
ARG_CONTAINERS + ")")
public String version;
@Parameter(names = {ARG_COMPONENTS}, variableArity = true,
description = "Filter containers by component names (used with " +
ARG_CONTAINERS + ")")
public Set<String> components = new HashSet<>(0);
/**
* Get the min #of params expected
* @return the min number of params in the {@link #parameters} field
*/
public int getMinParams() {
return 0;
}
@Override
public int getMaxParams() {
return 1;
}
}

218
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
View File

@ -1,218 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import org.apache.hadoop.yarn.service.exceptions.UsageException;
import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_REGISTRY;
import static org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
import java.io.File;
/**
* Registry actions
*
* --instance {app name}, if a / is in it, refers underneath?
* --dest {destfile}
* --list : list instances of slider service
* --listfiles
*/
@Parameters(commandNames = {ACTION_REGISTRY},
commandDescription = DESCRIBE_ACTION_REGISTRY)
public class ActionRegistryArgs extends AbstractActionArgs {
public static final String USAGE =
"Usage: " + SliderActions.ACTION_REGISTRY
+ " ("
+ Arguments.ARG_LIST + "|"
+ Arguments.ARG_LISTCONF + "|"
+ Arguments.ARG_LISTEXP + "|"
+ Arguments.ARG_LISTFILES + "|"
+ Arguments.ARG_GETCONF + "|"
+ Arguments.ARG_GETEXP + "> "
+ Arguments.ARG_NAME + " <name> "
+ " )"
+ "[" + Arguments.ARG_VERBOSE + "] "
+ "[" + Arguments.ARG_USER + "] "
+ "[" + Arguments.ARG_OUTPUT + " <filename> ] "
+ "[" + Arguments.ARG_SERVICETYPE + " <servicetype> ] "
+ "[" + Arguments.ARG_FORMAT + " <xml|json|properties>] "
+ System.getProperty("line.separator")
+ "Arguments.ARG_GETEXP only supports " + Arguments.ARG_FORMAT + " json"
;
public ActionRegistryArgs() {
}
public ActionRegistryArgs(String name) {
this.name = name;
}
@Override
public String getActionName() {
return ACTION_REGISTRY;
}
/**
* Get the min #of params expected
* @return the min number of params in the {@link #parameters} field
*/
@Override
public int getMinParams() {
return 0;
}
@Parameter(names = {ARG_LIST},
description = "list services")
public boolean list;
@Parameter(names = {ARG_LISTCONF},
description = "list configurations")
public boolean listConf;
@Parameter(names = {ARG_GETCONF},
description = "get configuration")
public String getConf;
@Parameter(names = {ARG_LISTEXP},
description = "list exports")
public boolean listExports;
@Parameter(names = {ARG_GETEXP},
description = "get export")
public String getExport;
@Parameter(names = {ARG_LISTFILES},
description = "list files")
public String listFiles;
@Parameter(names = {ARG_GETFILES},
description = "get files")
public String getFiles;
//--format
@Parameter(names = ARG_FORMAT,
description = "Format for a response: <xml|json|properties>")
public String format = ConfigFormat.XML.toString() ;
@Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT, ARG_DEST},
description = "Output destination")
public File out;
@Parameter(names = {ARG_NAME},
description = "name of an instance")
public String name;
@Parameter(names = {ARG_SERVICETYPE},
description = "optional service type")
public String serviceType = YarnServiceConstants.APP_TYPE;
@Parameter(names = {ARG_VERBOSE},
description = "verbose output")
public boolean verbose;
@Parameter(names = {ARG_INTERNAL},
description = "fetch internal registry entries")
public boolean internal;
@Parameter(names = {ARG_USER},
description = "the name of the user whose service is being resolved")
public String user;
/**
* validate health of all the different operations
* @throws BadCommandArgumentsException
*/
@Override
public void validate() throws BadCommandArgumentsException, UsageException {
super.validate();
//verify that at most one of the operations is set
int gets = s(getConf) + s(getFiles) + s(getExport);
int lists = s(list) + s(listConf) + s(listFiles) + s(listExports);
int set = lists + gets;
if (set > 1) {
throw new UsageException(USAGE);
}
if (out != null && ( set == 0)) {
throw new UsageException("output path"
+ " is only supported on 'get' operations: ");
}
if (!list && !is(name)) {
throw new UsageException("Argument " + ARG_NAME
+" missing: ");
}
}
private int s(String arg) {
return is(arg) ? 1 : 0;
}
private boolean is(String arg) {
return arg != null;
}
private int s(boolean arg) {
return arg ? 1 : 0;
}
private String ifdef(String arg, boolean val) {
return val ? (arg + " "): "";
}
private String ifdef(String arg, String val) {
if (is(val)) {
return arg + " " + val + " ";
} else {
return "";
}
}
@Override
public String toString() {
final StringBuilder sb =
new StringBuilder(ACTION_REGISTRY);
sb.append(' ');
sb.append(ifdef(ARG_LIST, list));
sb.append(ifdef(ARG_LISTCONF, listConf));
sb.append(ifdef(ARG_LISTFILES, listFiles));
sb.append(ifdef(ARG_GETCONF, getConf));
sb.append(ifdef(ARG_GETFILES, getFiles));
sb.append(ifdef(ARG_NAME, name));
sb.append(ifdef(ARG_SERVICETYPE, serviceType));
sb.append(ifdef(ARG_VERBOSE, verbose));
sb.append(ifdef(ARG_INTERNAL, internal));
if (out != null) {
sb.append(ifdef(ARG_OUTPUT, out.toString()));
}
sb.append(ifdef(ARG_FORMAT, format));
return sb.toString();
}
}

153
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java
View File

@ -1,153 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import org.apache.hadoop.yarn.service.exceptions.UsageException;
import java.io.File;
import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_RESOLVE;
import static org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
/**
* Resolve registry entries
*
* --path {path}
* --out {destfile}
* --verbose
* --list
*/
@Parameters(commandNames = {ACTION_RESOLVE},
commandDescription = DESCRIBE_ACTION_REGISTRY)
public class ActionResolveArgs extends AbstractActionArgs {
public static final String USAGE =
"Usage: " + SliderActions.ACTION_RESOLVE
+ " "
+ ARG_PATH + " <path> "
+ "[" + ARG_LIST + "] "
+ "[" + ARG_OUTPUT + " <filename> ] "
+ "[" + ARG_DESTDIR + " <directory> ] "
;
public ActionResolveArgs() {
}
@Override
public String getActionName() {
return ACTION_RESOLVE;
}
/**
* Get the min #of params expected
* @return the min number of params in the {@link #parameters} field
*/
@Override
public int getMinParams() {
return 0;
}
@Parameter(names = {ARG_LIST},
description = "list services")
public boolean list;
@Parameter(names = {ARG_PATH},
description = "resolve a path")
public String path;
@Parameter(names = {ARG_DESTDIR},
description = "destination directory for operations")
public File destdir;
@Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
description = "dest file")
public File out;
@Override
public String toString() {
final StringBuilder sb =
new StringBuilder(ACTION_RESOLVE).append(" ");
sb.append(ARG_PATH).append(" ").append(path).append(" ");
if (list) {
sb.append(ARG_LIST).append(" ");
}
if (destdir != null) {
sb.append(ARG_DESTDIR).append(" ").append(destdir).append(" ");
}
if (out != null) {
sb.append(ARG_OUTPUT).append(" ").append(out).append(" ");
}
return sb.toString();
}
@Override
public void validate() throws BadCommandArgumentsException, UsageException {
super.validate();
if (StringUtils.isEmpty(path)) {
throw new BadCommandArgumentsException("Missing mandatory argument "
+ ARG_PATH);
}
if (list && out != null) {
throw new BadCommandArgumentsException("Argument "
+ ARG_OUTPUT +
" not supported for " + ARG_LIST);
}
if (out != null && destdir != null) {
throw new BadCommandArgumentsException(
ARG_OUTPUT + " and " + ARG_DESTDIR + " cannot be used together"
);
}
}
public String getPath() {
return path;
}
public void setPath(String path) {
this.path = path;
}
public boolean isList() {
return list;
}
public void setList(boolean list) {
this.list = list;
}
public File getDestdir() {
return destdir;
}
public void setDestdir(File destdir) {
this.destdir = destdir;
}
public File getOut() {
return out;
}
public void setOut(File out) {
this.out = out;
}
}

70
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResourceArgs.java
View File

@ -1,70 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
import org.apache.hadoop.yarn.service.client.params.SliderActions;
@Parameters(commandNames = { SliderActions.ACTION_RESOURCE},
commandDescription = SliderActions.DESCRIBE_ACTION_RESOURCE)
public class ActionResourceArgs extends AbstractActionArgs {
@Override
public String getActionName() {
return SliderActions.ACTION_RESOURCE;
}
@Parameter(names = {ARG_INSTALL},
description = "Install the resource(s)")
public boolean install;
@Parameter(names = {ARG_DELETE},
description = "Delete the file")
public boolean delete;
@Parameter(names = {ARG_LIST},
description = "List of installed files")
public boolean list;
@Parameter(names = {ARG_RESOURCE},
description = "Name of the file or directory")
public String resource;
@Parameter(names = {ARG_DESTDIR},
description = "The name of the folder in which to store the resources")
public String folder;
@Parameter(names = {ARG_OVERWRITE}, description = "Overwrite existing resource(s)")
public boolean overwrite = false;
/**
* Get the min #of params expected
* @return the min number of params in the {@link #parameters} field
*/
public int getMinParams() {
return 0;
}
@Override
public int getMaxParams() {
return 3;
}
}

51
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionStatusArgs.java
View File

@ -1,51 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
import org.apache.hadoop.yarn.service.client.params.SliderActions;
@Parameters(commandNames = { SliderActions.ACTION_STATUS},
commandDescription = SliderActions.DESCRIBE_ACTION_STATUS)
public class ActionStatusArgs extends AbstractActionArgs {
@Override
public String getActionName() {
return SliderActions.ACTION_STATUS;
}
@Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
description = "Output file for the status information")
public String output;
@Parameter(names = {ARG_LIFETIME},
description = "Lifetime of the service from the time of request")
public boolean lifetime;
public String getOutput() {
return output;
}
public void setOutput(String output) {
this.output = output;
}
}

67
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionThawArgs.java
View File

@ -1,67 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.beust.jcommander.ParametersDelegate;
import java.io.File;
@Parameters(commandNames = { SliderActions.ACTION_START },
commandDescription = SliderActions.DESCRIBE_ACTION_THAW)
public class ActionThawArgs extends AbstractActionArgs implements
WaitTimeAccessor,
LaunchArgsAccessor {
@Override
public String getActionName() {
return SliderActions.ACTION_START;
}
@Override
public int getWaittime() {
return launchArgs.getWaittime();
}
@ParametersDelegate
LaunchArgsDelegate launchArgs = new LaunchArgsDelegate();
@Parameter(names = {ARG_LIFETIME},
description = "Life time of the service since service started at"
+ " running state")
public long lifetime;
@Override
public String getRmAddress() {
return launchArgs.getRmAddress();
}
@Override
public void setWaittime(int waittime) {
launchArgs.setWaittime(waittime);
}
@Override
public File getOutputFile() {
return launchArgs.getOutputFile();
}
}

78
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionTokensArgs.java
View File

@ -1,78 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import org.apache.hadoop.yarn.service.exceptions.UsageException;
import java.io.File;
@Parameters(commandNames = { SliderActions.ACTION_TOKENS},
commandDescription = "save tokens to a file or list tokens in a file")
public class ActionTokensArgs extends AbstractActionArgs {
public static final String DUPLICATE_ARGS = "Only one of " +
ARG_SOURCE + " and " + ARG_OUTPUT + " allowed";
public static final String MISSING_KT_PROVIDER =
"Both " + ARG_KEYTAB + " and " + ARG_PRINCIPAL
+ " must be provided";
@Override
public String getActionName() {
return SliderActions.ACTION_TOKENS;
}
@Parameter(names = {ARG_OUTPUT},
description = "File to write")
public File output;
@Parameter(names = {ARG_SOURCE},
description = "source file")
public File source;
@Parameter(names = {ARG_KEYTAB}, description = "keytab to use")
public File keytab;
@Parameter(names = {ARG_PRINCIPAL}, description = "principal to log in from a keytab")
public String principal="";
/**
* Get the min #of params expected
* @return the min number of params in the {@link #parameters} field
*/
public int getMinParams() {
return 0;
}
@Override
public void validate() throws BadCommandArgumentsException, UsageException {
super.validate();
if (output != null && source != null) {
throw new BadCommandArgumentsException(DUPLICATE_ARGS);
}
// this is actually a !xor
if (keytab != null ^ !principal.isEmpty()) {
throw new BadCommandArgumentsException(MISSING_KT_PROVIDER);
}
}
}

37
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
View File

@ -1,37 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
@Parameters(commandNames = { SliderActions.ACTION_UPDATE},
commandDescription = SliderActions.DESCRIBE_ACTION_UPDATE)
public class ActionUpdateArgs extends AbstractActionArgs {
@Parameter(names = {
ARG_LIFETIME }, description = "Lifetime of the service from the time of request")
public long lifetime;
@Override
public String getActionName() {
return SliderActions.ACTION_UPDATE;
}
}

156
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
View File

@ -1,156 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* Static argument manipulation operations
*/
public class ArgOps {
private static final Logger
log = LoggerFactory.getLogger(ArgOps.class);
/**
* create a 3-tuple
*/
public static List<Object> triple(String msg, int min, int max) {
List<Object> l = new ArrayList<>(3);
l.add(msg);
l.add(min);
l.add(max);
return l;
}
public static void applyFileSystemBinding(String filesystemBinding,
Configuration conf) {
if (filesystemBinding != null) {
//filesystem argument was set -this overwrites any defaults in the
//configuration
FileSystem.setDefaultUri(conf, filesystemBinding);
}
}
public static void splitPairs(Collection<String> pairs,
Map<String, String> dest) {
for (String prop : pairs) {
String[] keyval = prop.split("=", 2);
if (keyval.length == 2) {
dest.put(keyval[0], keyval[1]);
}
}
}
public static void applyDefinitions(Map<String, String> definitionMap,
Configuration conf) {
for (Map.Entry<String, String> entry : definitionMap.entrySet()) {
String key = entry.getKey();
String val = entry.getValue();
log.debug("configuration[{}]<=\"{}\"", key, val);
conf.set(key, val, "command line");
}
}
/**
* Create a map from a tuple list like ['worker','2','master','1] into a map
* ['worker':'2',"master":'1'];
* Duplicate entries also trigger errors
* @param description description for errors
* @param list list to conver to tuples
* @return the map of key value pairs -unordered.
* @throws BadCommandArgumentsException odd #of arguments received
*/
public static Map<String, String> convertTupleListToMap(String description,
List<String> list) throws
BadCommandArgumentsException {
Map<String, String> results = new HashMap<>();
if (list != null && !list.isEmpty()) {
int size = list.size();
if (size % 2 != 0) {
//odd number of elements, not permitted
throw new BadCommandArgumentsException(
ErrorStrings.ERROR_PARSE_FAILURE + description);
}
for (int count = 0; count < size; count += 2) {
String key = list.get(count);
String val = list.get(count + 1);
if (results.get(key) != null) {
throw new BadCommandArgumentsException(
ErrorStrings.ERROR_DUPLICATE_ENTRY + description
+ ": " + key);
}
results.put(key, val);
}
}
return results;
}
/**
* Create a map from a tuple list like
* ['worker','heapsize','5G','master','heapsize','2M'] into a map
* ['worker':'2',"master":'1'];
* Duplicate entries also trigger errors
* @throws BadCommandArgumentsException odd #of arguments received
*/
public static Map<String, Map<String, String>> convertTripleListToMaps(String description,
List<String> list) throws BadCommandArgumentsException {
Map<String, Map<String, String>> results = new HashMap<>();
if (list != null && !list.isEmpty()) {
int size = list.size();
if (size % 3 != 0) {
//wrong number of elements, not permitted
throw new BadCommandArgumentsException(
ErrorStrings.ERROR_PARSE_FAILURE + description);
}
for (int count = 0; count < size; count += 3) {
String role = list.get(count);
String key = list.get(count + 1);
String val = list.get(count + 2);
Map<String, String> roleMap = results.get(role);
if (roleMap == null) {
//demand create new role map
roleMap = new HashMap<>();
results.put(role, roleMap);
}
if (roleMap.get(key) != null) {
throw new BadCommandArgumentsException(
ErrorStrings.ERROR_DUPLICATE_ENTRY + description
+ ": for key " + key + " under " + role);
}
roleMap.put(key, val);
}
}
return results;
}
}

103
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
View File

@ -1,103 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
/**
* Here are all the arguments that may be parsed by the client or server
* command lines.
*
* Important: Please keep the main list in alphabetical order
* so it is easier to see what arguments are there
*/
public interface Arguments {
String ARG_FILE = "--file";
String ARG_FILE_SHORT = "-f";
String ARG_BASE_PATH = "--basepath";
String ARG_COMPONENT = "--component";
String ARG_COMPONENT_SHORT = "--comp";
String ARG_COMPONENTS = "--components";
String ARG_COMP_OPT= "--compopt";
String ARG_COMP_OPT_SHORT = "--co";
String ARG_CONFIG = "--config";
String ARG_CONTAINERS = "--containers";
String ARG_DEBUG = "--debug";
String ARG_DEFINE = "-D";
String ARG_DELETE = "--delete";
String ARG_DEST = "--dest";
String ARG_DESTDIR = "--destdir";
String ARG_EXAMPLE = "--example";
String ARG_EXAMPLE_SHORT = "-e";
String ARG_FOLDER = "--folder";
String ARG_FORCE = "--force";
String ARG_FORMAT = "--format";
String ARG_GETCONF = "--getconf";
String ARG_GETEXP = "--getexp";
String ARG_GETFILES = "--getfiles";
String ARG_HELP = "--help";
String ARG_IMAGE = "--image";
String ARG_INSTALL = "--install";
String ARG_INTERNAL = "--internal";
String ARG_KEYLEN = "--keylen";
String ARG_KEYTAB = "--keytab";
String ARG_KEYTABINSTALL = ARG_INSTALL;
String ARG_KEYTABDELETE = ARG_DELETE;
String ARG_KEYTABLIST = "--list";
String ARG_LIST = "--list";
String ARG_LISTCONF = "--listconf";
String ARG_LISTEXP = "--listexp";
String ARG_LISTFILES = "--listfiles";
String ARG_LIVE = "--live";
String ARG_MANAGER = "--manager";
String ARG_MANAGER_SHORT = "--m";
String ARG_MESSAGE = "--message";
String ARG_NAME = "--name";
String ARG_OPTION = "--option";
String ARG_OPTION_SHORT = "-O";
String ARG_OUTPUT = "--out";
String ARG_OUTPUT_SHORT = "-o";
String ARG_OVERWRITE = "--overwrite";
String ARG_PACKAGE = "--package";
String ARG_PATH = "--path";
String ARG_PRINCIPAL = "--principal";
String ARG_QUEUE = "--queue";
String ARG_SHORT_QUEUE = "-q";
String ARG_LIFETIME = "--lifetime";
String ARG_RESOURCE = "--resource";
String ARG_RESOURCE_MANAGER = "--rm";
String ARG_SECURE = "--secure";
String ARG_SERVICETYPE = "--servicetype";
String ARG_SERVICES = "--services";
String ARG_SOURCE = "--source";
String ARG_STATE = "--state";
String ARG_SYSPROP = "-S";
String ARG_USER = "--user";
String ARG_UPLOAD = "--upload";
String ARG_VERBOSE = "--verbose";
String ARG_VERSION = "--version";
String ARG_WAIT = "--wait";
/*
STOP: DO NOT ADD YOUR ARGUMENTS HERE. GO BACK AND INSERT THEM IN THE
RIGHT PLACE IN THE LIST
*/
// Tha path in hdfs to be read by Service AM
String ARG_SERVICE_DEF_PATH = "-cluster-uri";
}

193
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
View File

@ -1,193 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.service.utils.SliderUtils;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
import org.apache.hadoop.yarn.service.exceptions.SliderException;
import java.util.Collection;
/**
* Client CLI Args
*/
public class ClientArgs extends CommonArgs {
// =========================================================
// Keep all of these in alphabetical order. Thanks.
// =========================================================
private final ActionBuildArgs actionBuildArgs = new ActionBuildArgs();
private final ActionClientArgs actionClientArgs = new ActionClientArgs();
private final ActionCreateArgs actionCreateArgs = new ActionCreateArgs();
private final ActionDependencyArgs actionDependencyArgs = new ActionDependencyArgs();
private final ActionDestroyArgs actionDestroyArgs = new ActionDestroyArgs();
private final ActionExistsArgs actionExistsArgs = new ActionExistsArgs();
private final ActionFlexArgs actionFlexArgs = new ActionFlexArgs();
private final ActionFreezeArgs actionFreezeArgs = new ActionFreezeArgs();
private final ActionHelpArgs actionHelpArgs = new ActionHelpArgs();
private final ActionKeytabArgs actionKeytabArgs = new ActionKeytabArgs();
private final ActionListArgs actionListArgs = new ActionListArgs();
private final ActionRegistryArgs actionRegistryArgs = new ActionRegistryArgs();
private final ActionResolveArgs actionResolveArgs = new ActionResolveArgs();
private final ActionResourceArgs actionResourceArgs = new ActionResourceArgs();
private final ActionStatusArgs actionStatusArgs = new ActionStatusArgs();
private final ActionThawArgs actionThawArgs = new ActionThawArgs();
private final ActionTokensArgs actionTokenArgs = new ActionTokensArgs();
private final ActionUpdateArgs actionUpdateArgs = new ActionUpdateArgs();
public ClientArgs(String[] args) {
super(args);
}
public ClientArgs(Collection args) {
super(args);
}
@Override
protected void addActionArguments() {
addActions(
actionBuildArgs,
actionCreateArgs,
actionDependencyArgs,
actionDestroyArgs,
actionFlexArgs,
actionFreezeArgs,
actionHelpArgs,
actionStatusArgs,
actionThawArgs
);
}
@Override
public void applyDefinitions(Configuration conf) throws
BadCommandArgumentsException {
super.applyDefinitions(conf);
}
public ActionBuildArgs getActionBuildArgs() {
return actionBuildArgs;
}
public ActionUpdateArgs getActionUpdateArgs() {
return actionUpdateArgs;
}
public ActionCreateArgs getActionCreateArgs() {
return actionCreateArgs;
}
public ActionDependencyArgs getActionDependencyArgs() {
return actionDependencyArgs;
}
public ActionFlexArgs getActionFlexArgs() {
return actionFlexArgs;
}
/**
* Look at the chosen action and bind it as the core action for the operation.
* @throws SliderException bad argument or similar
*/
@Override
public void applyAction() throws SliderException {
String action = getAction();
if (SliderUtils.isUnset(action)) {
action = ACTION_HELP;
}
switch (action) {
case ACTION_BUILD:
bindCoreAction(actionBuildArgs);
break;
case ACTION_CREATE:
bindCoreAction(actionCreateArgs);
break;
case ACTION_STOP:
bindCoreAction(actionFreezeArgs);
break;
case ACTION_START:
bindCoreAction(actionThawArgs);
break;
case ACTION_DEPENDENCY:
bindCoreAction(actionDependencyArgs);
break;
case ACTION_DESTROY:
bindCoreAction(actionDestroyArgs);
break;
case ACTION_EXISTS:
bindCoreAction(actionExistsArgs);
break;
case ACTION_FLEX:
bindCoreAction(actionFlexArgs);
break;
case ACTION_HELP:
bindCoreAction(actionHelpArgs);
break;
case ACTION_KEYTAB:
bindCoreAction(actionKeytabArgs);
break;
case ACTION_LIST:
bindCoreAction(actionListArgs);
break;
case ACTION_REGISTRY:
bindCoreAction(actionRegistryArgs);
break;
case ACTION_RESOLVE:
bindCoreAction(actionResolveArgs);
break;
case ACTION_RESOURCE:
bindCoreAction(actionResourceArgs);
break;
case ACTION_STATUS:
bindCoreAction(actionStatusArgs);
break;
case ACTION_TOKENS:
bindCoreAction(actionTokenArgs);
break;
case ACTION_UPDATE:
bindCoreAction(actionUpdateArgs);
break;
default:
throw new BadCommandArgumentsException(ErrorStrings.ERROR_UNKNOWN_ACTION
+ " " + action);
}
}
}

245
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
View File

@ -1,245 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.ParameterDescription;
import com.beust.jcommander.ParameterException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.service.utils.SliderUtils;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
import org.apache.hadoop.yarn.service.exceptions.SliderException;
import org.apache.hadoop.yarn.service.exceptions.UsageException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* This class contains the common argument set for all tne entry points,
* and the core parsing logic to verify that the action is on the list
* of allowed actions -and that the remaining number of arguments is
* in the range allowed
*/
public abstract class CommonArgs extends ArgOps implements SliderActions,
Arguments {
protected static final Logger log = LoggerFactory.getLogger(CommonArgs.class);
private static final int DIFF_BETWEEN_DESCIPTION_AND_COMMAND_NAME = 30;
@Parameter(names = ARG_HELP, help = true)
public boolean help;
/**
-D name=value
Define an HBase configuration option which overrides any options in
the configuration XML files of the image or in the image configuration
directory. The values will be persisted.
Configuration options are only passed to the cluster when creating or reconfiguring a cluster.
*/
public Map<String, String> definitionMap = new HashMap<String, String>();
/**
* System properties
*/
public Map<String, String> syspropsMap = new HashMap<String, String>();
/**
* fields
*/
public final JCommander commander;
private final String[] args;
private AbstractActionArgs coreAction;
/**
* get the name: relies on arg 1 being the cluster name in all operations
* @return the name argument, null if there is none
*/
public String getClusterName() {
return coreAction.getServiceName();
}
protected CommonArgs(String[] args) {
this.args = args;
commander = new JCommander(this);
}
protected CommonArgs(Collection args) {
List<String> argsAsStrings = SliderUtils.collectionToStringList(args);
this.args = argsAsStrings.toArray(new String[argsAsStrings.size()]);
commander = new JCommander(this);
}
public String usage() {
return usage(this, null);
}
public static String usage(CommonArgs serviceArgs, String commandOfInterest) {
String result = null;
StringBuilder helperMessage = new StringBuilder();
if (commandOfInterest == null) {
// JCommander.usage is too verbose for a command with many options like
// slider no short version of that is found Instead, we compose our msg by
helperMessage.append("\nUsage: service COMMAND [options]\n");
helperMessage.append("where COMMAND is one of\n");
for (String jcommand : serviceArgs.commander.getCommands().keySet()) {
helperMessage.append(String.format("\t%-"
+ DIFF_BETWEEN_DESCIPTION_AND_COMMAND_NAME + "s%s", jcommand,
serviceArgs.commander.getCommandDescription(jcommand) + "\n"));
}
helperMessage
.append("Most commands print help when invoked without parameters or with --help");
result = helperMessage.toString();
} else {
helperMessage.append("\nUsage: service ").append(commandOfInterest);
helperMessage.append(serviceArgs.coreAction.getMinParams() > 0 ? " <service>" : "");
helperMessage.append("\n");
for (ParameterDescription paramDesc : serviceArgs.commander.getCommands()
.get(commandOfInterest).getParameters()) {
String optional = paramDesc.getParameter().required() ? " (required)"
: " (optional)";
String paramName = paramDesc.getParameterized().getType() == Boolean.TYPE ? paramDesc
.getLongestName() : paramDesc.getLongestName() + " <"
+ paramDesc.getParameterized().getName() + ">";
helperMessage.append(String.format("\t%-"
+ DIFF_BETWEEN_DESCIPTION_AND_COMMAND_NAME + "s%s", paramName,
paramDesc.getDescription() + optional + "\n"));
result = helperMessage.toString();
}
}
return result;
}
/**
* Parse routine -includes registering the action-specific argument classes
* and postprocess it
* @throws SliderException on any problem
*/
public void parse() throws SliderException {
addActionArguments();
try {
commander.parse(args);
} catch (ParameterException e) {
commander.usage(args[0]);
throw e;
}
//now copy back to this class some of the attributes that are common to all
//actions
postProcess();
}
protected void addActions(Object... actions) {
for (Object action : actions) {
commander.addCommand(action);
}
}
/**
* Override point to add a set of actions
*/
protected void addActionArguments() {
}
/**
* validate args via {@link #validate()}
* then postprocess the arguments
*/
public void postProcess() throws SliderException {
applyAction();
validate();
//apply entry set
for (Map.Entry<String, String> entry : syspropsMap.entrySet()) {
System.setProperty(entry.getKey(), entry.getValue());
}
}
/**
* Implementors must implement their action apply routine here
*/
public abstract void applyAction() throws SliderException;
/**
* Bind the core action; this extracts any attributes that are used
* across routines
* @param action action to bind
*/
protected void bindCoreAction(AbstractActionArgs action) {
coreAction = action;
splitPairs(coreAction.definitions, definitionMap);
splitPairs(coreAction.sysprops, syspropsMap);
}
/**
* Validate the arguments against the action requested
*/
public void validate() throws BadCommandArgumentsException, UsageException {
if (coreAction == null) {
throw new UsageException(ErrorStrings.ERROR_NO_ACTION + usage());
}
log.debug("action={}", getAction());
// let the action validate itself
try {
coreAction.validate();
} catch (BadCommandArgumentsException e) {
String badArgMsgBuilder =
e.getMessage() + System.lineSeparator() + usage(this,
coreAction.getActionName());
throw new BadCommandArgumentsException(badArgMsgBuilder);
}
}
/**
* Apply all the definitions on the command line to the configuration
* @param conf config
*/
public void applyDefinitions(Configuration conf) throws
BadCommandArgumentsException {
applyDefinitions(definitionMap, conf);
}
public boolean isDebug() {
return coreAction.debug;
}
public String getAction() {
return commander.getParsedCommand();
}
}

52
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ComponentArgsDelegate.java
View File

@ -1,52 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
public class ComponentArgsDelegate extends AbstractArgsDelegate {
/**
* This is a listing of the roles to create
*/
@Parameter(names = {ARG_COMPONENT, ARG_COMPONENT_SHORT},
arity = 2,
description = "--component <name> <count> e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3",
splitter = DontSplitArguments.class)
public List<String> componentTuples = new ArrayList<>(0);
/**
* Get the role mapping (may be empty, but never null)
* @return role mapping
* @throws BadCommandArgumentsException parse problem
*/
public Map<String, String> getComponentMap() throws BadCommandArgumentsException {
return convertTupleListToMap("component", componentTuples);
}
public List<String> getComponentTuples() {
return componentTuples;
}
}

34
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/DontSplitArguments.java
View File

@ -1,34 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.converters.IParameterSplitter;
import java.util.ArrayList;
import java.util.List;
public class DontSplitArguments implements IParameterSplitter {
@Override
public List<String> split(String value) {
List<String> list = new ArrayList<>(1);
list.add(value);
return list;
}
}

30
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsAccessor.java
View File

@ -1,30 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import java.io.File;
/**
* Launch args for create and start and anything else that can start something
*/
public interface LaunchArgsAccessor extends WaitTimeAccessor {
String getRmAddress();
File getOutputFile();
}

51
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/LaunchArgsDelegate.java
View File

@ -1,51 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import java.io.File;
/**
* Any launch-time args
*/
public class LaunchArgsDelegate extends WaitArgsDelegate implements
LaunchArgsAccessor {
//TODO: do we need this?
@Parameter(names = ARG_RESOURCE_MANAGER,
description = "Resource manager hostname:port ",
required = false)
private String rmAddress;
@Override
public String getRmAddress() {
return rmAddress;
}
@Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
description = "output file for any service report")
public File outputFile;
@Override
public File getOutputFile() {
return outputFile;
}
}

66
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/OptionArgsDelegate.java
View File

@ -1,66 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* Delegate for application and resource options.
*/
public class OptionArgsDelegate extends AbstractArgsDelegate {
/**
* Options key value.
*/
@Parameter(names = {ARG_OPTION, ARG_OPTION_SHORT}, arity = 2,
description = ARG_OPTION + "<name> <value>",
splitter = DontSplitArguments.class)
public List<String> optionTuples = new ArrayList<>(0);
/**
* All the app component option triples.
*/
@Parameter(names = {ARG_COMP_OPT, ARG_COMP_OPT_SHORT}, arity = 3,
description = "Component option " + ARG_COMP_OPT +
" <component> <name> <option>",
splitter = DontSplitArguments.class)
public List<String> compOptTriples = new ArrayList<>(0);
public Map<String, String> getOptionsMap() throws
BadCommandArgumentsException {
return convertTupleListToMap(ARG_OPTION, optionTuples);
}
/**
* Get the role heap mapping (may be empty, but never null).
* @return role heap mapping
* @throws BadCommandArgumentsException parse problem
*/
public Map<String, Map<String, String>> getCompOptionMap()
throws BadCommandArgumentsException {
return convertTripleListToMaps(ARG_COMP_OPT, compOptTriples);
}
}

34
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/PathArgumentConverter.java
View File

@ -1,34 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.converters.BaseConverter;
import org.apache.hadoop.fs.Path;
public class PathArgumentConverter extends BaseConverter<Path> {
public PathArgumentConverter(String optionName) {
super(optionName);
}
@Override
public Path convert(String value) {
return new Path(value);
}
}

50
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMArgs.java
View File

@ -1,50 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
/**
* Parameters sent by the Client to the AM
*/
public class ServiceAMArgs extends CommonArgs {
ServiceAMCreateAction createAction = new ServiceAMCreateAction();
public ServiceAMArgs(String[] args) {
super(args);
}
@Override
protected void addActionArguments() {
addActions(createAction);
}
// This is the path in hdfs to the service definition JSON file
public String getServiceDefPath() {
return createAction.serviceDefPath;
}
/**
* Am binding is simple: there is only one action
*/
@Override
public void applyAction() {
bindCoreAction(createAction);
}
}

37
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ServiceAMCreateAction.java
View File

@ -1,37 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
@Parameters(commandNames = { SliderActions.ACTION_CREATE},
commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
public class ServiceAMCreateAction extends AbstractActionArgs {
@Override
public String getActionName() {
return SliderActions.ACTION_CREATE;
}
@Parameter(names = ARG_SERVICE_DEF_PATH,
description = "Path to the service definition JSON file", required = true)
public String serviceDefPath;
}

79
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
View File

@ -1,79 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
/**
* Actions by client
*/
public interface SliderActions {
String ACTION_BUILD = "build";
String ACTION_CLIENT = "client";
String ACTION_CREATE = "create";
String ACTION_DEPENDENCY = "dependency";
String ACTION_UPDATE = "update";
String ACTION_UPGRADE = "upgrade";
String ACTION_DESTROY = "destroy";
String ACTION_EXISTS = "exists";
String ACTION_FLEX = "flex";
String ACTION_STOP = "stop";
String ACTION_HELP = "help";
String ACTION_INSTALL_KEYTAB = "install-keytab";
String ACTION_KEYTAB = "keytab";
String ACTION_LIST = "list";
String ACTION_REGISTRY = "registry";
String ACTION_RESOLVE = "resolve";
String ACTION_RESOURCE = "resource";
String ACTION_STATUS = "status";
String ACTION_START = "start";
String ACTION_TOKENS = "tokens";
String DESCRIBE_ACTION_BUILD =
"Build a service specification, but do not start it";
String DESCRIBE_ACTION_CREATE =
"Create a service, it's equivalent to first invoke build and then start";
String DESCRIBE_ACTION_DEPENDENCY =
"Yarn service framework dependency (libraries) management";
String DESCRIBE_ACTION_UPDATE =
"Update template for service";
String DESCRIBE_ACTION_UPGRADE =
"Rolling upgrade/downgrade the component/containerto a newer/previous version";
String DESCRIBE_ACTION_DESTROY =
"Destroy a stopped service, service must be stopped first before destroying.";
String DESCRIBE_ACTION_EXISTS =
"Probe for a service running";
String DESCRIBE_ACTION_FLEX = "Flex a service's component by increasing or decreasing the number of containers.";
String DESCRIBE_ACTION_FREEZE =
"Stop a running service";
String DESCRIBE_ACTION_KDIAG = "Diagnose Kerberos problems";
String DESCRIBE_ACTION_HELP = "Print help information";
String DESCRIBE_ACTION_LIST =
"List running services";
String DESCRIBE_ACTION_REGISTRY =
"Query the registry of a service";
String DESCRIBE_ACTION_STATUS =
"Get the status of a service";
String DESCRIBE_ACTION_THAW =
"Start a service with pre-built specification or a previously stopped service";
String DESCRIBE_ACTION_CLIENT = "Install the service client in the specified directory or obtain a client keystore or truststore";
String DESCRIBE_ACTION_KEYTAB = "Manage a Kerberos keytab file (install, delete, list) in the sub-folder 'keytabs' of the user's Slider base directory";
String DESCRIBE_ACTION_RESOURCE = "Manage a file (install, delete, list) in the 'resources' sub-folder of the user's Slider base directory";
}

42
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitArgsDelegate.java
View File

@ -1,42 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
public class WaitArgsDelegate extends AbstractArgsDelegate implements
WaitTimeAccessor {
//--wait [timeout]
@Parameter(names = {ARG_WAIT},
description = "time to wait for an action to complete")
public int waittime = 0;
@Override
public int getWaittime() {
return waittime;
}
@Override
public void setWaittime(int waittime) {
this.waittime = waittime;
}
}

24
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/WaitTimeAccessor.java
View File

@ -1,24 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
public interface WaitTimeAccessor {
int getWaittime();
void setWaittime(int waittime);
}

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
View File

@ -29,8 +29,6 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.service.client.params.Arguments;
import org.apache.hadoop.yarn.service.client.params.SliderActions;
import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
@ -484,8 +482,8 @@ public static ClasspathConstructor buildClasspath(String sliderConfDir,
classpath.addLibDir(YarnServiceConstants.DEPENDENCY_LOCALIZED_DIR_LINK);
} else {
log.info(
"For faster submission of apps, upload dependencies using cmd {} {}",
SliderActions.ACTION_DEPENDENCY, Arguments.ARG_UPLOAD);
"For faster submission of apps, upload dependencies using cmd " +
"enableFastLaunch");
}
classpath.addRemoteClasspathEnvVar();
classpath.append(ApplicationConstants.Environment.HADOOP_CONF_DIR.$$());

19
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestBuildExternalComponents.java
View File

@ -19,10 +19,10 @@
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.client.api.AppAdminClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.conf.ExampleAppJson;
import org.apache.hadoop.yarn.service.client.params.ClientArgs;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.junit.After;
@ -36,7 +36,6 @@
import java.util.List;
import java.util.Set;
import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_FILE;
import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
/**
@ -60,18 +59,10 @@ private static void checkComponentNames(List<Component> components,
// 2. check component names
private void buildAndCheckComponents(String appName, String appDef,
SliderFileSystem sfs, Set<String> names) throws Throwable {
String[] args =
{ "build", ARG_FILE, ExampleAppJson.resourceName(appDef) };
ClientArgs clientArgs = new ClientArgs(args);
clientArgs.parse();
ServiceCLI cli = new ServiceCLI() {
@Override protected void createServiceClient() {
client = new ServiceClient();
client.init(conf);
client.start();
}
};
cli.exec(clientArgs);
AppAdminClient client = AppAdminClient.createAppAdminClient(AppAdminClient
.DEFAULT_TYPE, conf);
client.actionSave(ExampleAppJson.resourceName(appDef), null, null,
null);
// verify generated conf
List<Component> components =

125
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
View File

@ -20,88 +20,71 @@
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.yarn.client.cli.ApplicationCLI;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.service.ClientAMProtocol;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.client.params.ClientArgs;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.conf.ExampleAppJson;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.apache.hadoop.yarn.util.Records;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import static org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS;
import static org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS;
import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_FILE;
import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
import static org.mockito.Mockito.*;
public class TestServiceCLI {
private static final Logger LOG = LoggerFactory.getLogger(TestServiceCLI
.class);
protected Configuration conf = new YarnConfiguration();
private Configuration conf = new YarnConfiguration();
private File basedir;
private ServiceCLI cli;
private SliderFileSystem fs;
private String basedirProp;
private void buildApp(String appDef) throws Throwable {
String[] args =
{ "build", ARG_FILE, ExampleAppJson.resourceName(appDef)};
ClientArgs clientArgs = new ClientArgs(args);
clientArgs.parse();
cli.exec(clientArgs);
private void runCLI(String[] args) throws Exception {
LOG.info("running CLI: yarn {}", Arrays.asList(args));
ApplicationCLI cli = new ApplicationCLI();
cli.setSysOutPrintStream(System.out);
cli.setSysErrPrintStream(System.err);
int res = ToolRunner.run(cli, ApplicationCLI.preProcessArgs(args));
cli.stop();
}
private void buildApp(String serviceName, String appDef) throws Throwable {
String[] args = {"app", "-D", basedirProp, "-save", serviceName,
ExampleAppJson.resourceName(appDef)};
runCLI(args);
}
private void buildApp(String serviceName, String appDef, String lifetime,
String queue) throws Throwable {
String[] args = {"app", "-D", basedirProp, "-save", serviceName,
ExampleAppJson.resourceName(appDef), "-updateLifetime", lifetime,
"-changeQueue", queue};
runCLI(args);
}
@Before
public void setup() throws Throwable {
basedir = new File("target", "apps");
basedirProp = YARN_SERVICE_BASE_PATH + "=" + basedir.getAbsolutePath();
conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath());
conf.setLong(RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, 0);
conf.setLong(RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, 1);
conf.setInt(
CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
conf.setInt(CommonConfigurationKeysPublic.
IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 0);
fs = new SliderFileSystem(conf);
if (basedir.exists()) {
FileUtils.deleteDirectory(basedir);
} else {
basedir.mkdirs();
}
// create a CLI and skip connection to AM
cli = new ServiceCLI() {
@Override protected void createServiceClient() {
client = new ServiceClient() {
@Override
protected void serviceInit(Configuration configuration)
throws Exception {
super.serviceInit(conf);
yarnClient = spy(yarnClient);
ApplicationReport report = Records.newRecord(ApplicationReport.class);
report.setYarnApplicationState(YarnApplicationState.RUNNING);
report.setHost("localhost");
doReturn(report).when(yarnClient).getApplicationReport(anyObject());
}
@Override
protected ClientAMProtocol createAMProxy(String host, int port)
throws IOException {
return mock(ClientAMProtocol.class);
}
};
client.init(conf);
client.start();
}
};
}
@After
@ -111,41 +94,31 @@ public void tearDown() throws IOException {
}
}
// Test flex components count are persisted.
@Test
public void testFlexComponents() throws Throwable {
// currently can only test building apps, since that is the only
// operation that doesn't require an RM
// TODO: expand CLI test to try other commands
String serviceName = "app-1";
buildApp(ExampleAppJson.APP_JSON);
checkCompCount("master",serviceName, 1L);
buildApp(serviceName, ExampleAppJson.APP_JSON);
checkApp(serviceName, "master", 1L, 3600L, null);
// increase by 2
String[] flexUpArgs = {"flex", serviceName, "--component", "master" , "+2"};
ClientArgs clientArgs = new ClientArgs(flexUpArgs);
clientArgs.parse();
cli.exec(clientArgs);
checkCompCount("master", serviceName, 3L);
// decrease by 1
String[] flexDownArgs = {"flex", serviceName, "--component", "master", "-1"};
clientArgs = new ClientArgs(flexDownArgs);
clientArgs.parse();
cli.exec(clientArgs);
checkCompCount("master", serviceName, 2L);
String[] flexAbsoluteArgs = {"flex", serviceName, "--component", "master", "10"};
clientArgs = new ClientArgs(flexAbsoluteArgs);
clientArgs.parse();
cli.exec(clientArgs);
checkCompCount("master", serviceName, 10L);
serviceName = "app-2";
buildApp(serviceName, ExampleAppJson.APP_JSON, "1000", "qname");
checkApp(serviceName, "master", 1L, 1000L, "qname");
}
private void checkCompCount(String compName, String serviceName, long count)
throws IOException {
List<Component> components =
ServiceApiUtil.getComponents(fs, serviceName);
private void checkApp(String serviceName, String compName, long count, Long
lifetime, String queue) throws IOException {
Service service = ServiceApiUtil.loadService(fs, serviceName);
Assert.assertEquals(serviceName, service.getName());
Assert.assertEquals(lifetime, service.getLifetime());
Assert.assertEquals(queue, service.getQueue());
List<Component> components = service.getComponents();
for (Component component : components) {
if (component.getName().equals(compName)) {
Assert.assertEquals(count, component.getNumberOfContainers().longValue());
Assert.assertEquals(count, component.getNumberOfContainers()
.longValue());
return;
}
}

222
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java Normal file
View File

@ -0,0 +1,222 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.client.api;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import java.io.IOException;
import java.util.Map;
/**
* Client for managing applications.
*/
@Public
@Unstable
public abstract class AppAdminClient extends CompositeService {
public static final String YARN_APP_ADMIN_CLIENT_PREFIX = "yarn" +
".application.admin.client.class.";
public static final String DEFAULT_TYPE = "yarn-service";
public static final String DEFAULT_CLASS_NAME = "org.apache.hadoop.yarn" +
".service.client.ServiceClient";
@Private
protected AppAdminClient() {
super(AppAdminClient.class.getName());
}
/**
* <p>
* Create a new instance of AppAdminClient.
* </p>
*
* @param appType application type
* @param conf configuration
* @return app admin client
*/
@Public
@Unstable
public static AppAdminClient createAppAdminClient(String appType,
Configuration conf) {
Map<String, String> clientClassMap =
conf.getPropsWithPrefix(YARN_APP_ADMIN_CLIENT_PREFIX);
if (!clientClassMap.containsKey(DEFAULT_TYPE)) {
clientClassMap.put(DEFAULT_TYPE, DEFAULT_CLASS_NAME);
}
if (!clientClassMap.containsKey(appType)) {
throw new IllegalArgumentException("App admin client class name not " +
"specified for type " + appType);
}
String clientClassName = clientClassMap.get(appType);
Class<? extends AppAdminClient> clientClass;
try {
clientClass = (Class<? extends AppAdminClient>) Class.forName(
clientClassName);
} catch (ClassNotFoundException e) {
throw new YarnRuntimeException("Invalid app admin client class", e);
}
AppAdminClient appAdminClient = ReflectionUtils.newInstance(clientClass,
conf);
appAdminClient.init(conf);
appAdminClient.start();
return appAdminClient;
}
/**
* <p>
* Launch a new YARN application.
* </p>
*
* @param fileName specification of application
* @param appName name of the application
* @param lifetime lifetime of the application
* @param queue queue of the application
* @return exit code
* @throws IOException IOException
* @throws YarnException exception in client or server
*/
@Public
@Unstable
public abstract int actionLaunch(String fileName, String appName, Long
lifetime, String queue) throws IOException, YarnException;
/**
* <p>
* Stop a YARN application (attempt to stop gracefully before killing the
* application). In the case of a long-running service, the service may be
* restarted later.
* </p>
*
* @param appName the name of the application
* @return exit code
* @throws IOException IOException
* @throws YarnException exception in client or server
*/
@Public
@Unstable
public abstract int actionStop(String appName) throws IOException,
YarnException;
/**
* <p>
* Start a YARN application from a previously saved specification. In the
* case of a long-running service, the service must have been previously
* launched/started and then stopped, or previously saved but not started.
* </p>
*
* @param appName the name of the application
* @return exit code
* @throws IOException IOException
* @throws YarnException exception in client or server
*/
@Public
@Unstable
public abstract int actionStart(String appName) throws IOException,
YarnException;
/**
* <p>
* Save the specification for a YARN application / long-running service.
* The application may be started later.
* </p>
*
* @param fileName specification of application to save
* @param appName name of the application
* @param lifetime lifetime of the application
* @param queue queue of the application
* @return exit code
* @throws IOException IOException
* @throws YarnException exception in client or server
*/
@Public
@Unstable
public abstract int actionSave(String fileName, String appName, Long
lifetime, String queue) throws IOException, YarnException;
/**
* <p>
* Remove the specification and all application data for a YARN application.
* The application cannot be running.
* </p>
*
* @param appName the name of the application
* @return exit code
* @throws IOException IOException
* @throws YarnException exception in client or server
*/
@Public
@Unstable
public abstract int actionDestroy(String appName) throws IOException,
YarnException;
/**
* <p>
* Change the number of running containers for a component of a YARN
* application / long-running service.
* </p>
*
* @param appName the name of the application
* @param componentCounts map of component name to new component count or
* amount to change existing component count (e.g.
* 5, +5, -5)
* @return exit code
* @throws IOException IOException
* @throws YarnException exception in client or server
*/
@Public
@Unstable
public abstract int actionFlex(String appName, Map<String, String>
componentCounts) throws IOException, YarnException;
/**
* <p>
* Upload AM dependencies to HDFS. This makes future application launches
* faster since the dependencies do not have to be uploaded on each launch.
* </p>
*
* @return exit code
* @throws IOException IOException
* @throws YarnException exception in client or server
*/
@Public
@Unstable
public abstract int enableFastLaunch() throws IOException,
YarnException;
/**
* <p>
* Get detailed status string for a YARN application.
* </p>
*
* @param applicationId application id
* @return status string
* @throws IOException IOException
* @throws YarnException exception in client or server
*/
@Public
@Unstable
public abstract String getStatusString(String applicationId) throws
IOException, YarnException;
}

322
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
View File

@ -23,12 +23,7 @@
import java.io.PrintWriter;
import java.nio.charset.Charset;
import java.text.DecimalFormat;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.*;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
@ -54,6 +49,7 @@
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.SignalContainerCommand;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.AppAdminClient;
import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
@ -85,6 +81,7 @@ public class ApplicationCLI extends YarnCLI {
"%30s\t%20s\t%20s\t%20s\t%20s\t%20s\t%35s"
+ System.getProperty("line.separator");
public static final String APP = "app";
public static final String APPLICATION = "application";
public static final String APPLICATION_ATTEMPT = "applicationattempt";
public static final String CONTAINER = "container";
@ -93,22 +90,52 @@ public class ApplicationCLI extends YarnCLI {
public static final String UPDATE_LIFETIME = "updateLifetime";
public static final String CHANGE_APPLICATION_QUEUE = "changeQueue";
// app admin options
public static final String LAUNCH_CMD = "launch";
public static final String STOP_CMD = "stop";
public static final String START_CMD = "start";
public static final String SAVE_CMD = "save";
public static final String DESTROY_CMD = "destroy";
public static final String FLEX_CMD = "flex";
public static final String COMPONENT = "component";
public static final String ENABLE_FAST_LAUNCH = "enableFastLaunch";
private static String firstArg = null;
private boolean allAppStates;
public static void main(String[] args) throws Exception {
ApplicationCLI cli = new ApplicationCLI();
cli.setSysOutPrintStream(System.out);
cli.setSysErrPrintStream(System.err);
int res = ToolRunner.run(cli, args);
int res = ToolRunner.run(cli, preProcessArgs(args));
cli.stop();
System.exit(res);
}
@VisibleForTesting
public static String[] preProcessArgs(String[] args) {
if (args.length > 0) {
// first argument (app|application|applicationattempt|container) must
// be stripped off for GenericOptionsParser to work
firstArg = args[0];
return Arrays.copyOfRange(args, 1, args.length);
} else {
return args;
}
}
@Override
public int run(String[] args) throws Exception {
Options opts = new Options();
String title = null;
if (args.length > 0 && args[0].equalsIgnoreCase(APPLICATION)) {
if (firstArg != null) {
title = firstArg;
} else if (args.length > 0) {
title = args[0];
}
if (title != null && (title.equalsIgnoreCase(APPLICATION) || title
.equalsIgnoreCase(APP))) {
title = APPLICATION;
opts.addOption(STATUS_CMD, true,
"Prints the status of the application.");
@ -168,8 +195,52 @@ public int run(String[] args) throws Exception {
opts.getOption(UPDATE_PRIORITY).setArgName("Priority");
opts.getOption(UPDATE_LIFETIME).setArgName("Timeout");
opts.getOption(CHANGE_APPLICATION_QUEUE).setArgName("Queue Name");
} else if (args.length > 0 && args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) {
title = APPLICATION_ATTEMPT;
opts.addOption(LAUNCH_CMD, true, "Launches application from " +
"specification file (saves specification and starts application). " +
"Options -updateLifetime and -changeQueue can be specified to alter" +
" the values provided in the file. Supports -appTypes option to " +
"specify which client implementation to use.");
opts.addOption(STOP_CMD, true, "Stops application gracefully (may be " +
"started again later). If name is provided, appType must be " +
"provided unless it is the default yarn-service. If ID is provided," +
" the appType will be looked up. Supports -appTypes option to " +
"specify which client implementation to use.");
opts.addOption(START_CMD, true, "Starts a previously saved " +
"application. Supports -appTypes option to specify which client " +
"implementation to use.");
opts.addOption(SAVE_CMD, true, "Saves specification file for " +
"an application. Options -updateLifetime and -changeQueue can be " +
"specified to alter the values provided in the file. Supports " +
"-appTypes option to specify which client implementation to use.");
opts.addOption(DESTROY_CMD, true, "Destroys a saved application " +
"specification and removes all application data permanently. " +
"Supports -appTypes option to specify which client implementation " +
"to use.");
opts.addOption(FLEX_CMD, true, "Changes number of " +
"running containers for a component of an application / " +
"long-running service. Requires -component option. If name is " +
"provided, appType must be provided unless it is the default " +
"yarn-service. If ID is provided, the appType will be looked up. " +
"Supports -appTypes option to specify which client implementation " +
"to use.");
opts.addOption(COMPONENT, true, "Works with -flex option to change " +
"the number of components/containers running for an application / " +
"long-running service. Supports absolute or relative changes, such " +
"as +1, 2, or -3.");
opts.addOption(ENABLE_FAST_LAUNCH, false, "Uploads AM dependencies " +
"to HDFS to make future launches faster. Supports -appTypes option" +
" to specify which client implementation to use.");
opts.getOption(LAUNCH_CMD).setArgName("Application Name> <File Name");
opts.getOption(LAUNCH_CMD).setArgs(2);
opts.getOption(START_CMD).setArgName("Application Name");
opts.getOption(STOP_CMD).setArgName("Application Name or ID");
opts.getOption(SAVE_CMD).setArgName("Application Name> <File Name");
opts.getOption(SAVE_CMD).setArgs(2);
opts.getOption(DESTROY_CMD).setArgName("Application Name");
opts.getOption(FLEX_CMD).setArgName("Application Name or ID");
opts.getOption(COMPONENT).setArgName("Component Name> <Count");
opts.getOption(COMPONENT).setArgs(2);
} else if (title != null && title.equalsIgnoreCase(APPLICATION_ATTEMPT)) {
opts.addOption(STATUS_CMD, true,
"Prints the status of the application attempt.");
opts.addOption(LIST_CMD, true,
@ -179,8 +250,7 @@ public int run(String[] args) throws Exception {
opts.getOption(STATUS_CMD).setArgName("Application Attempt ID");
opts.getOption(LIST_CMD).setArgName("Application ID");
opts.getOption(FAIL_CMD).setArgName("Application Attempt ID");
} else if (args.length > 0 && args[0].equalsIgnoreCase(CONTAINER)) {
title = CONTAINER;
} else if (title != null && title.equalsIgnoreCase(CONTAINER)) {
opts.addOption(STATUS_CMD, true,
"Prints the status of the container.");
opts.addOption(LIST_CMD, true,
@ -205,23 +275,53 @@ public int run(String[] args) throws Exception {
printUsage(title, opts);
return exitCode;
}
if (cliParser.hasOption(STATUS_CMD)) {
if (args.length != 3) {
String[] unparsedArgs = cliParser.getArgs();
if (firstArg == null) {
if (unparsedArgs.length != 1) {
printUsage(title, opts);
return exitCode;
}
if (args[0].equalsIgnoreCase(APPLICATION)) {
exitCode = printApplicationReport(cliParser.getOptionValue(STATUS_CMD));
} else if (args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) {
} else {
if (unparsedArgs.length != 0) {
printUsage(title, opts);
return exitCode;
}
}
if (cliParser.hasOption(STATUS_CMD)) {
if (hasAnyOtherCLIOptions(cliParser, opts, STATUS_CMD)) {
printUsage(title, opts);
return exitCode;
}
if (title.equalsIgnoreCase(APPLICATION) ||
title.equalsIgnoreCase(APP)) {
ApplicationReport report = printApplicationReport(cliParser
.getOptionValue(STATUS_CMD));
if (report == null) {
exitCode = -1;
} else {
exitCode = 0;
String appType = report.getApplicationType();
try {
AppAdminClient client = AppAdminClient.createAppAdminClient(appType,
getConf());
sysout.println("Detailed Application Status :");
sysout.println(client.getStatusString(cliParser.getOptionValue(
STATUS_CMD)));
} catch (IllegalArgumentException e) {
// app type does not have app admin client implementation
}
}
} else if (title.equalsIgnoreCase(APPLICATION_ATTEMPT)) {
exitCode = printApplicationAttemptReport(cliParser
.getOptionValue(STATUS_CMD));
} else if (args[0].equalsIgnoreCase(CONTAINER)) {
} else if (title.equalsIgnoreCase(CONTAINER)) {
exitCode = printContainerReport(cliParser.getOptionValue(STATUS_CMD));
}
return exitCode;
} else if (cliParser.hasOption(LIST_CMD)) {
if (args[0].equalsIgnoreCase(APPLICATION)) {
if (title.equalsIgnoreCase(APPLICATION) ||
title.equalsIgnoreCase(APP)) {
allAppStates = false;
Set<String> appTypes = new HashSet<String>();
if (cliParser.hasOption(APP_TYPE_CMD)) {
@ -272,21 +372,21 @@ public int run(String[] args) throws Exception {
}
}
listApplications(appTypes, appStates, appTags);
} else if (args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) {
if (args.length != 3) {
} else if (title.equalsIgnoreCase(APPLICATION_ATTEMPT)) {
if (hasAnyOtherCLIOptions(cliParser, opts, LIST_CMD)) {
printUsage(title, opts);
return exitCode;
}
listApplicationAttempts(cliParser.getOptionValue(LIST_CMD));
} else if (args[0].equalsIgnoreCase(CONTAINER)) {
if (args.length != 3) {
} else if (title.equalsIgnoreCase(CONTAINER)) {
if (hasAnyOtherCLIOptions(cliParser, opts, LIST_CMD)) {
printUsage(title, opts);
return exitCode;
}
listContainers(cliParser.getOptionValue(LIST_CMD));
}
} else if (cliParser.hasOption(KILL_CMD)) {
if (args.length < 3 || hasAnyOtherCLIOptions(cliParser, opts, KILL_CMD)) {
if (hasAnyOtherCLIOptions(cliParser, opts, KILL_CMD)) {
printUsage(title, opts);
return exitCode;
}
@ -299,7 +399,7 @@ public int run(String[] args) throws Exception {
moveApplicationAcrossQueues(cliParser.getOptionValue(MOVE_TO_QUEUE_CMD),
cliParser.getOptionValue(QUEUE_CMD));
} else if (cliParser.hasOption(FAIL_CMD)) {
if (!args[0].equalsIgnoreCase(APPLICATION_ATTEMPT)) {
if (!title.equalsIgnoreCase(APPLICATION_ATTEMPT)) {
printUsage(title, opts);
return exitCode;
}
@ -314,6 +414,103 @@ public int run(String[] args) throws Exception {
}
updateApplicationPriority(cliParser.getOptionValue(APP_ID),
cliParser.getOptionValue(UPDATE_PRIORITY));
} else if (cliParser.hasOption(SIGNAL_CMD)) {
if (hasAnyOtherCLIOptions(cliParser, opts, SIGNAL_CMD)) {
printUsage(title, opts);
return exitCode;
}
final String[] signalArgs = cliParser.getOptionValues(SIGNAL_CMD);
final String containerId = signalArgs[0];
SignalContainerCommand command =
SignalContainerCommand.OUTPUT_THREAD_DUMP;
if (signalArgs.length == 2) {
command = SignalContainerCommand.valueOf(signalArgs[1]);
}
signalToContainer(containerId, command);
} else if (cliParser.hasOption(LAUNCH_CMD)) {
if (hasAnyOtherCLIOptions(cliParser, opts, LAUNCH_CMD, APP_TYPE_CMD,
UPDATE_LIFETIME, CHANGE_APPLICATION_QUEUE)) {
printUsage(title, opts);
return exitCode;
}
String appType = getSingleAppTypeFromCLI(cliParser);
Long lifetime = null;
if (cliParser.hasOption(UPDATE_LIFETIME)) {
lifetime = Long.parseLong(cliParser.getOptionValue(UPDATE_LIFETIME));
}
String queue = null;
if (cliParser.hasOption(CHANGE_APPLICATION_QUEUE)) {
queue = cliParser.getOptionValue(CHANGE_APPLICATION_QUEUE);
}
String[] nameAndFile = cliParser.getOptionValues(LAUNCH_CMD);
return AppAdminClient.createAppAdminClient(appType, getConf())
.actionLaunch(nameAndFile[1], nameAndFile[0], lifetime, queue);
} else if (cliParser.hasOption(STOP_CMD)) {
if (hasAnyOtherCLIOptions(cliParser, opts, STOP_CMD, APP_TYPE_CMD)) {
printUsage(title, opts);
return exitCode;
}
String[] appNameAndType = getAppNameAndType(cliParser, STOP_CMD);
return AppAdminClient.createAppAdminClient(appNameAndType[1], getConf())
.actionStop(appNameAndType[0]);
} else if (cliParser.hasOption(START_CMD)) {
if (hasAnyOtherCLIOptions(cliParser, opts, START_CMD, APP_TYPE_CMD)) {
printUsage(title, opts);
return exitCode;
}
String appType = getSingleAppTypeFromCLI(cliParser);
return AppAdminClient.createAppAdminClient(appType, getConf())
.actionStart(cliParser.getOptionValue(START_CMD));
} else if (cliParser.hasOption(SAVE_CMD)) {
if (hasAnyOtherCLIOptions(cliParser, opts, SAVE_CMD, APP_TYPE_CMD,
UPDATE_LIFETIME, CHANGE_APPLICATION_QUEUE)) {
printUsage(title, opts);
return exitCode;
}
String appType = getSingleAppTypeFromCLI(cliParser);
Long lifetime = null;
if (cliParser.hasOption(UPDATE_LIFETIME)) {
lifetime = Long.parseLong(cliParser.getOptionValue(UPDATE_LIFETIME));
}
String queue = null;
if (cliParser.hasOption(CHANGE_APPLICATION_QUEUE)) {
queue = cliParser.getOptionValue(CHANGE_APPLICATION_QUEUE);
}
String[] nameAndFile = cliParser.getOptionValues(SAVE_CMD);
return AppAdminClient.createAppAdminClient(appType, getConf())
.actionSave(nameAndFile[1], nameAndFile[0], lifetime, queue);
} else if (cliParser.hasOption(DESTROY_CMD)) {
if (hasAnyOtherCLIOptions(cliParser, opts, DESTROY_CMD, APP_TYPE_CMD)) {
printUsage(title, opts);
return exitCode;
}
String appType = getSingleAppTypeFromCLI(cliParser);
return AppAdminClient.createAppAdminClient(appType, getConf())
.actionDestroy(cliParser.getOptionValue(DESTROY_CMD));
} else if (cliParser.hasOption(FLEX_CMD)) {
if (!cliParser.hasOption(COMPONENT) ||
hasAnyOtherCLIOptions(cliParser, opts, FLEX_CMD, COMPONENT,
APP_TYPE_CMD)) {
printUsage(title, opts);
return exitCode;
}
String[] rawCounts = cliParser.getOptionValues(COMPONENT);
Map<String, String> counts = new HashMap<>(rawCounts.length/2);
for (int i = 0; i < rawCounts.length - 1; i+=2) {
counts.put(rawCounts[i], rawCounts[i+1]);
}
String[] appNameAndType = getAppNameAndType(cliParser, FLEX_CMD);
return AppAdminClient.createAppAdminClient(appNameAndType[1], getConf())
.actionFlex(appNameAndType[0], counts);
} else if (cliParser.hasOption(ENABLE_FAST_LAUNCH)) {
String appType = getSingleAppTypeFromCLI(cliParser);
if (hasAnyOtherCLIOptions(cliParser, opts, ENABLE_FAST_LAUNCH,
APP_TYPE_CMD)) {
printUsage(title, opts);
return exitCode;
}
return AppAdminClient.createAppAdminClient(appType, getConf())
.enableFastLaunch();
} else if (cliParser.hasOption(UPDATE_LIFETIME)) {
if (!cliParser.hasOption(APP_ID)) {
printUsage(title, opts);
@ -332,19 +529,6 @@ public int run(String[] args) throws Exception {
}
moveApplicationAcrossQueues(cliParser.getOptionValue(APP_ID),
cliParser.getOptionValue(CHANGE_APPLICATION_QUEUE));
} else if (cliParser.hasOption(SIGNAL_CMD)) {
if (args.length < 3 || args.length > 4) {
printUsage(title, opts);
return exitCode;
}
final String[] signalArgs = cliParser.getOptionValues(SIGNAL_CMD);
final String containerId = signalArgs[0];
SignalContainerCommand command =
SignalContainerCommand.OUTPUT_THREAD_DUMP;
if (signalArgs.length == 2) {
command = SignalContainerCommand.valueOf(signalArgs[1]);
}
signalToContainer(containerId, command);
} else {
syserr.println("Invalid Command Usage : ");
printUsage(title, opts);
@ -352,6 +536,47 @@ public int run(String[] args) throws Exception {
return 0;
}
private ApplicationReport getApplicationReport(ApplicationId applicationId)
throws IOException, YarnException {
ApplicationReport appReport = null;
try {
appReport = client.getApplicationReport(applicationId);
} catch (ApplicationNotFoundException e) {
throw new YarnException("Application with id '" + applicationId
+ "' doesn't exist in RM or Timeline Server.");
}
return appReport;
}
private String[] getAppNameAndType(CommandLine cliParser, String option)
throws IOException, YarnException {
String applicationIdOrName = cliParser.getOptionValue(option);
try {
ApplicationId id = ApplicationId.fromString(applicationIdOrName);
ApplicationReport report = getApplicationReport(id);
return new String[]{report.getName(), report.getApplicationType()};
} catch (IllegalArgumentException e) {
// assume CLI option provided the app name
// and read appType from command line since id wasn't provided
String appType = getSingleAppTypeFromCLI(cliParser);
return new String[]{applicationIdOrName, appType};
}
}
private static String getSingleAppTypeFromCLI(CommandLine cliParser) {
if (cliParser.hasOption(APP_TYPE_CMD)) {
String[] types = cliParser.getOptionValues(APP_TYPE_CMD);
if (types != null) {
for (String type : types) {
if (!type.trim().isEmpty()) {
return StringUtils.toLowerCase(type).trim();
}
}
}
}
return AppAdminClient.DEFAULT_TYPE;
}
private void updateApplicationTimeout(String applicationId,
ApplicationTimeoutType timeoutType, long timeoutInSec)
throws YarnException, IOException {
@ -572,7 +797,7 @@ private void listApplications(Set<String> appTypes,
/**
* Kills applications with the application id as appId
*
* @param Array of applicationIds
* @param applicationIds Array of applicationIds
* @return errorCode
* @throws YarnException
* @throws IOException
@ -663,10 +888,10 @@ private void failApplicationAttempt(String attemptId) throws YarnException,
* Prints the application report for an application id.
*
* @param applicationId
* @return exitCode
* @return ApplicationReport
* @throws YarnException
*/
private int printApplicationReport(String applicationId)
private ApplicationReport printApplicationReport(String applicationId)
throws YarnException, IOException {
ApplicationReport appReport = null;
try {
@ -675,7 +900,7 @@ private int printApplicationReport(String applicationId)
} catch (ApplicationNotFoundException e) {
sysout.println("Application with id '" + applicationId
+ "' doesn't exist in RM or Timeline Server.");
return -1;
return null;
}
// Use PrintWriter.println, which uses correct platform line ending.
ByteArrayOutputStream baos = new ByteArrayOutputStream();
@ -739,11 +964,11 @@ private int printApplicationReport(String applicationId)
+ "' doesn't exist in RM.");
appReportStr.close();
sysout.println(baos.toString("UTF-8"));
return -1;
return null;
}
appReportStr.close();
sysout.println(baos.toString("UTF-8"));
return 0;
return appReport;
}
private void printResourceUsage(PrintWriter appReportStr,
@ -856,11 +1081,12 @@ private void updateApplicationPriority(String applicationId, String priority)
@SuppressWarnings("unchecked")
private boolean hasAnyOtherCLIOptions(CommandLine cliParser, Options opts,
String excludeOption) {
String... excludeOptions) {
Collection<Option> ops = opts.getOptions();
Set<String> excludeSet = new HashSet<>(Arrays.asList(excludeOptions));
for (Option op : ops) {
// Skip exclude option from the option list
if (op.getOpt().equals(excludeOption)) {
// Skip exclude options from the option list
if (excludeSet.contains(op.getOpt())) {
continue;
}
if (cliParser.hasOption(op.getOpt())) {

158
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
View File

@ -2025,49 +2025,121 @@ private String createApplicationCLIHelpMessage() throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw = new PrintWriter(baos);
pw.println("usage: application");
pw.println(" -appId <Application ID> Specify Application Id to be operated");
pw.println(" -appStates <States> Works with -list to filter applications");
pw.println(" based on input comma-separated list of");
pw.println(" application states. The valid application");
pw.println(" state can be one of the following:");
pw.println(" ALL,NEW,NEW_SAVING,SUBMITTED,ACCEPTED,RUN");
pw.println(" NING,FINISHED,FAILED,KILLED");
pw.println(" -appTags <Tags> Works with -list to filter applications");
pw.println(" based on input comma-separated list of");
pw.println(" application tags.");
pw.println(" -appTypes <Types> Works with -list to filter applications");
pw.println(" based on input comma-separated list of");
pw.println(" application types.");
pw.println(" -changeQueue <Queue Name> Moves application to a new queue.");
pw.println(" ApplicationId can be passed using 'appId'");
pw.println(" option. 'movetoqueue' command is");
pw.println(" deprecated, this new command");
pw.println(" 'changeQueue' performs same");
pw.println(" functionality.");
pw.println(" -help Displays help for all commands.");
pw.println(" -kill <Application ID> Kills the application. Set of");
pw.println(" applications can be provided separated");
pw.println(" with space");
pw.println(" -list List applications. Supports optional use");
pw.println(" of -appTypes to filter applications based");
pw.println(" on application type, -appStates to filter");
pw.println(" applications based on application state");
pw.println(" and -appTags to filter applications based");
pw.println(" on application tag.");
pw.println(" -movetoqueue <Application ID> Moves the application to a different");
pw.println(" queue. Deprecated command. Use");
pw.println(" 'changeQueue' instead.");
pw.println(" -queue <Queue Name> Works with the movetoqueue command to");
pw.println(" specify which queue to move an");
pw.println(" application to.");
pw.println(" -status <Application ID> Prints the status of the application.");
pw.println(" -updateLifetime <Timeout> update timeout of an application from");
pw.println(" NOW. ApplicationId can be passed using");
pw.println(" 'appId' option. Timeout value is in");
pw.println(" seconds.");
pw.println(" -updatePriority <Priority> update priority of an application.");
pw.println(" ApplicationId can be passed using 'appId'");
pw.println(" option.");
pw.println(" -appId <Application ID> Specify Application Id to be");
pw.println(" operated");
pw.println(" -appStates <States> Works with -list to filter");
pw.println(" applications based on input");
pw.println(" comma-separated list of");
pw.println(" application states. The valid");
pw.println(" application state can be one of");
pw.println(" the following:");
pw.println(" ALL,NEW,NEW_SAVING,SUBMITTED,ACC");
pw.println(" EPTED,RUNNING,FINISHED,FAILED,KI");
pw.println(" LLED");
pw.println(" -appTags <Tags> Works with -list to filter");
pw.println(" applications based on input");
pw.println(" comma-separated list of");
pw.println(" application tags.");
pw.println(" -appTypes <Types> Works with -list to filter");
pw.println(" applications based on input");
pw.println(" comma-separated list of");
pw.println(" application types.");
pw.println(" -changeQueue <Queue Name> Moves application to a new");
pw.println(" queue. ApplicationId can be");
pw.println(" passed using 'appId' option.");
pw.println(" 'movetoqueue' command is");
pw.println(" deprecated, this new command");
pw.println(" 'changeQueue' performs same");
pw.println(" functionality.");
pw.println(" -component <Component Name> <Count> Works with -flex option to");
pw.println(" change the number of");
pw.println(" components/containers running");
pw.println(" for an application /");
pw.println(" long-running service. Supports");
pw.println(" absolute or relative changes,");
pw.println(" such as +1, 2, or -3.");
pw.println(" -destroy <Application Name> Destroys a saved application");
pw.println(" specification and removes all");
pw.println(" application data permanently.");
pw.println(" Supports -appTypes option to");
pw.println(" specify which client");
pw.println(" implementation to use.");
pw.println(" -enableFastLaunch Uploads AM dependencies to HDFS");
pw.println(" to make future launches faster.");
pw.println(" Supports -appTypes option to");
pw.println(" specify which client");
pw.println(" implementation to use.");
pw.println(" -flex <Application Name or ID> Changes number of running");
pw.println(" containers for a component of an");
pw.println(" application / long-running");
pw.println(" service. Requires -component");
pw.println(" option. If name is provided,");
pw.println(" appType must be provided unless");
pw.println(" it is the default yarn-service.");
pw.println(" If ID is provided, the appType");
pw.println(" will be looked up. Supports");
pw.println(" -appTypes option to specify");
pw.println(" which client implementation to");
pw.println(" use.");
pw.println(" -help Displays help for all commands.");
pw.println(" -kill <Application ID> Kills the application. Set of");
pw.println(" applications can be provided");
pw.println(" separated with space");
pw.println(" -launch <Application Name> <File Name> Launches application from");
pw.println(" specification file (saves");
pw.println(" specification and starts");
pw.println(" application). Options");
pw.println(" -updateLifetime and -changeQueue");
pw.println(" can be specified to alter the");
pw.println(" values provided in the file.");
pw.println(" Supports -appTypes option to");
pw.println(" specify which client");
pw.println(" implementation to use.");
pw.println(" -list List applications. Supports");
pw.println(" optional use of -appTypes to");
pw.println(" filter applications based on");
pw.println(" application type, -appStates to");
pw.println(" filter applications based on");
pw.println(" application state and -appTags");
pw.println(" to filter applications based on");
pw.println(" application tag.");
pw.println(" -movetoqueue <Application ID> Moves the application to a");
pw.println(" different queue. Deprecated");
pw.println(" command. Use 'changeQueue'");
pw.println(" instead.");
pw.println(" -queue <Queue Name> Works with the movetoqueue");
pw.println(" command to specify which queue");
pw.println(" to move an application to.");
pw.println(" -save <Application Name> <File Name> Saves specification file for an");
pw.println(" application. Options");
pw.println(" -updateLifetime and -changeQueue");
pw.println(" can be specified to alter the");
pw.println(" values provided in the file.");
pw.println(" Supports -appTypes option to");
pw.println(" specify which client");
pw.println(" implementation to use.");
pw.println(" -start <Application Name> Starts a previously saved");
pw.println(" application. Supports -appTypes");
pw.println(" option to specify which client");
pw.println(" implementation to use.");
pw.println(" -status <Application ID> Prints the status of the");
pw.println(" application.");
pw.println(" -stop <Application Name or ID> Stops application gracefully");
pw.println(" (may be started again later). If");
pw.println(" name is provided, appType must");
pw.println(" be provided unless it is the");
pw.println(" default yarn-service. If ID is");
pw.println(" provided, the appType will be");
pw.println(" looked up. Supports -appTypes");
pw.println(" option to specify which client");
pw.println(" implementation to use.");
pw.println(" -updateLifetime <Timeout> update timeout of an application");
pw.println(" from NOW. ApplicationId can be");
pw.println(" passed using 'appId' option.");
pw.println(" Timeout value is in seconds.");
pw.println(" -updatePriority <Priority> update priority of an");
pw.println(" application. ApplicationId can");
pw.println(" be passed using 'appId' option.");
pw.close();
String appsHelpStr = baos.toString("UTF-8");
return appsHelpStr;

86
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
View File

@ -37,25 +37,36 @@ User Commands
Commands useful for users of a Hadoop cluster.
### `application`
### `application` or `app`
Usage: `yarn application [options] `
Usage: `yarn app [options] `
| COMMAND\_OPTIONS | Description |
|:---- |:---- |
| -appId \<ApplicationId\> | Specify Application Id to be operated |
| -appStates \<States\> | Works with -list to filter applications based on input comma-separated list of application states. The valid application state can be one of the following:  ALL, NEW, NEW\_SAVING, SUBMITTED, ACCEPTED, RUNNING, FINISHED, FAILED, KILLED |
| -appTags \<Tags\> | Works with -list to filter applications based on input comma-separated list of application tags. |
| -appTypes \<Types\> | Works with -list to filter applications based on input comma-separated list of application types. |
| -help | Help |
| -list | Lists applications from the RM. Supports optional use of -appTypes to filter applications based on application type, and -appStates to filter applications based on application state. |
| -kill \<ApplicationId\> | Kills the application. |
| -movetoqueue \<Application Id\> | Moves the application to a different queue. |
| -changeQueue \<Queue Name\> | Moves application to a new queue. ApplicationId can be passed using 'appId' option. 'movetoqueue' command is deprecated, this new command 'changeQueue' performs same functionality. |
| -component \<Component Name\> \<Count\> | Works with -flex option to change the number of components/containers running for an application / long-running service. Supports absolute or relative changes, such as +1, 2, or -3. |
| -destroy \<Application Name\> | Destroys a saved application specification and removes all application data permanently. Supports -appTypes option to specify which client implementation to use. |
| -enableFastLaunch | Uploads AM dependencies to HDFS to make future launches faster. Supports -appTypes option to specify which client implementation to use. |
| -flex \<Application Name or ID\> | Changes number of running containers for a component of an application / long-running service. Requires -component option. If name is provided, appType must be provided unless it is the default yarn-service. If ID is provided, the appType will be looked up. Supports -appTypes option to specify which client implementation to use. |
| -help | Displays help for all commands. |
| -kill \<Application ID\> | Kills the application. Set of applications can be provided separated with space |
| -launch \<Application Name\> \<File Name\> | Launches application from specification file (saves specification and starts application). Options -updateLifetime and -changeQueue can be specified to alter the values provided in the file. Supports -appTypes option to specify which client implementation to use. |
| -list | List applications. Supports optional use of -appTypes to filter applications based on application type, -appStates to filter applications based on application state and -appTags to filter applications based on application tag. |
| -movetoqueue \<Application ID\> | Moves the application to a different queue. Deprecated command. Use 'changeQueue' instead. |
| -queue \<Queue Name\> | Works with the movetoqueue command to specify which queue to move an application to. |
| -save \<Application Name\> \<File Name\> | Saves specification file for an application. Options -updateLifetime and -changeQueue can be specified to alter the values provided in the file. Supports -appTypes option to specify which client implementation to use. |
| -start \<Application Name\> | Starts a previously saved application. Supports -appTypes option to specify which client implementation to use. |
| -status \<ApplicationId\> | Prints the status of the application. |
| -updateLifetime \<Timeout\> | Update application timeout (from the time of request) in seconds. ApplicationId can be specified using 'appId' option. |
| -stop \<Application Name or ID\> | Stops application gracefully (may be started again later). If name is provided, appType must be provided unless it is the default yarn-service. If ID is provided, the appType will be looked up. Supports -appTypes option to specify which client implementation to use. |
| -updateLifetime \<Timeout\> | Update timeout of an application from NOW. ApplicationId can be passed using 'appId' option. Timeout value is in seconds. |
| -updatePriority \<Priority\> | Update priority of an application. ApplicationId can be passed using 'appId' option. |
Prints application(s) report/kill application
Prints application(s) report/kill application/manage long running application
### `applicationattempt`
@ -69,61 +80,6 @@ Usage: `yarn applicationattempt [options] `
prints applicationattempt(s) report
### `service`
Usage `yarn service [sub-command] [service-name] [options]`
#### `SUB_COMMAND`
* `build`: Build a service with its specifications, but do not start it.
```
Usage: yarn service build [service-name] --file [file]
Fields:
service-name Optional. If specified, it will override the name in the service definition.
Options:
--file,-f The local path to the service definition file
```
* `create`: create a service, it's equivalent to first invoke build and then start.
```
Usage: yarn service create [service-name] --file [file]
Fields:
service-name Optional. If specified, it will override the name in the service definition.
Options:
--file,-f The local path to the service definition file.
--queue,-q The queue to which the service is submitted.
--example,-e The name of the example service such as:
Sleeper A simple service that launches a few non-docker sleep containers on YARN.
```
* `dependency`: Yarn service framework dependency (libraries) management.
```
Usage: yarn service dependency [options]
Option:
--upload Pre-upload the dependency jars onto HDFS to expediate service launch process.
```
* `destroy`: Destroy a stopped service, service must be stopped first before destroying.
```
Usage: yarn service destroy [service-name]
```
* `flex`: Flex a service's component by increasing or decreasing the number of containers.
```
Usage: yarn service flex [service-name] --component [component-name] [count]
Options:
--component [component-name] [count]
Specifies the component name and its number of containers. e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3.
```
* `status`: Get the status of a service.
```
Usage: yarn service status [service-name]
```
* `start`: Start a service with pre-built specification or a previously stopped service.
```
Usage: yarn service start [service-name]
```
### `classpath`
Usage: `yarn classpath [--glob |--jar <path> |-h |--help]`
@ -328,12 +284,6 @@ Usage: `yarn timelineserver`
Start the TimeLineServer
### apiserver
Usage: `yarn apiserver`
Start the API-server for deploying/managing services on YARN
### registrydns
Usage: `yarn registrydns`

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
View File

@ -100,16 +100,18 @@ ProxyPassReverse "/" "balancer://test/"
Then run the service with the command:
```
yarn service create [service-name] --example httpd
yarn app -launch <service-name> httpd
```
where `service-name` is optional. If omitted, it uses the name defined in the `Yarnfile`.
The last argument is either the path to a JSON specification of the service, or in this case, the name of an example service.
The directory where examples can be found can be configured by setting the YARN\_EXAMPLES\_DIR environment variable.
Once the service is running, navigate to `http://httpd-proxy-0.${SERVICE_NAME}.${USER}.${DOMAIN}:8080` to see the root page.
The pages should alternately show "Hello from httpd-0!" or "Hello from httpd-1!"
The individual httpd URLs can also be visited, `http://httpd-0.${SERVICE_NAME}.${USER}.${DOMAIN}:8080` and `http://httpd-1.${SERVICE_NAME}.${USER}.${DOMAIN}:8080`.
If unsure of your hostnames, visit the apiserver REST endpoint `http://<apiserver host>:9191/ws/v1/services/httpd-service`.
If unsure of your hostnames, visit the RM REST endpoint `http://<RM host>:8088/ws/v1/services/httpd-service`.
## Apache web server - httpd (without registry DNS)
@ -151,9 +153,9 @@ hdfs dfs -copyFromLocal ${HADOOP_YARN_HOME}/share/hadoop/yarn/yarn-service-examp
Then run the service with the command:
```
yarn service create [service-name] --example httpd-no-dns
yarn app -launch <service-name> httpd-no-dns
```
where `service-name` is optional. If omitted, it uses the name defined in the `Yarnfile`.
Look up your IPs at the apiserver REST endpoint `http://<apiserver host>:9191/ws/v1/services/httpd-service`.
Look up your IPs at the RM REST endpoint `http://<RM host>:8088/ws/v1/services/httpd-service`.
Then visit port 8080 for each IP to view the pages.

46
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
View File

@ -45,11 +45,11 @@ Below is a simple service definition that launches sleep containers on YARN by w
```
User can simply run a pre-built example service on YARN using below command:
```
yarn service create [service-name] --example [example-name]
yarn app -launch <service-name> <example-name>
```
e.g. Below command launches a `sleeper` service named as `my-sleeper` on YARN.
```
yarn service create my-sleeper --example sleeper
yarn app -launch my-sleeper sleeper
```
For launching docker based services using YARN Service framework, please refer to [API doc](YarnServiceAPI.md).
@ -58,51 +58,53 @@ Below steps walk you through deploying a services on YARN using CLI.
Refer to [Yarn Commands](../YarnCommands.md) for the full list of commands and options.
### Deploy a service
```
yarn service create --file ${PATH_TO_SERVICE_DEF_FILE}
yarn app -launch ${SERVICE_NAME} ${PATH_TO_SERVICE_DEF_FILE}
```
Params:
- SERVICE_NAME: The name of the service. Note that this needs to be unique across all running services.
- SERVICE_NAME: The name of the service. Note that this needs to be unique across running services for the current user.
- PATH_TO_SERVICE_DEF: The path to the service definition file in JSON format.
For example:
```
yarn service create --file /path/to/local/sleeper.json
yarn app -launch sleeper-service /path/to/local/sleeper.json
```
### Flex a component of a service
Increase or decrease the number of containers for a component.
```
yarn service flex ${SERVICE_NAME} --component ${COMPONENT_NAME} ${NUMBER_OF_CONTAINERS}
yarn app -flex ${SERVICE_NAME} -component ${COMPONENT_NAME} ${NUMBER_OF_CONTAINERS}
```
For example, for a service named `sleeper-service`:
Set the `sleeper` component to `2` containers (absolute number).
```
yarn service flex sleeper-service --component sleeper 2
yarn app -flex sleeper-service -component sleeper 2
```
Relative changes are also supported for the ${NUMBER_OF_CONTAINERS} in the flex command, such as +2 or -2.
### Stop a service
Stopping a service will stop all containers of the service and the ApplicationMaster, but does not delete the state of a service, such as the service root folder on hdfs.
```
yarn service stop ${SERVICE_NAME}
yarn app -stop ${SERVICE_NAME}
```
### Restart a stopped service
Restarting a stopped service is easy - just call start!
```
yarn service start ${SERVICE_NAME}
yarn app -start ${SERVICE_NAME}
```
### Destroy a service
In addition to stopping a service, it also deletes the service root folder on hdfs and the records in YARN Service Registry.
```
yarn service destroy ${SERVICE_NAME}
yarn app -destroy ${SERVICE_NAME}
```
## Manage services on YARN via REST API
YARN API Server REST API can be activated in two modes: embedded or standalone.
YARN API Server REST API can be activated as part of the ResourceManager.
### Start Embedded API-Server as part of ResourceManager
For running inside ResourceManager, add this property to `yarn-site.xml` and restart ResourceManager.
@ -119,13 +121,6 @@ For running inside ResourceManager, add this property to `yarn-site.xml` and res
Services can be deployed on YARN through the ResourceManager web endpoint.
### Start Standalone API-Server for deploying services on YARN
API server is the service that sits in front of YARN ResourceManager and lets users submit their service specs via HTTP.
```
yarn --daemon start apiserver
```
The above command starts the API Server on the localhost at port 9191 by default.
Refer to [API doc](YarnServiceAPI.md) for the detailed API specificatiosn.
### Deploy a service
@ -135,19 +130,14 @@ POST the aforementioned example service definition to the ResourceManager api-se
POST http://localhost:8088/ws/v1/services
```
Or standalone API server:
```
POST http://localhost:9191/ws/v1/services
```
### Get a service status
```
GET http://localhost:9191/ws/v1/services/${SERVICE_NAME}
GET http://localhost:8088/ws/v1/services/${SERVICE_NAME}
```
### Flex a component of a service
```
PUT http://localhost:9191/ws/v1/services/${SERVICE_NAME}/components/${COMPONENT_NAME}
PUT http://localhost:8088/ws/v1/services/${SERVICE_NAME}/components/${COMPONENT_NAME}
```
`PUT` Request body:
```
@ -168,7 +158,7 @@ For example:
Stopping a service will stop all containers of the service and the ApplicationMaster, but does not delete the state of a service, such as the service root folder on hdfs.
```
PUT http://localhost:9191/ws/v1/services/${SERVICE_NAME}
PUT http://localhost:8088/ws/v1/services/${SERVICE_NAME}
```
`PUT` Request body:
@ -183,7 +173,7 @@ PUT http://localhost:9191/ws/v1/services/${SERVICE_NAME}
Restarting a stopped service is easy.
```
PUT http://localhost:9191/ws/v1/services/${SERVICE_NAME}
PUT http://localhost:8088/ws/v1/services/${SERVICE_NAME}
```
`PUT` Request body:
@ -196,7 +186,7 @@ PUT http://localhost:9191/ws/v1/services/${SERVICE_NAME}
### Destroy a service
In addition to stopping a service, it also deletes the service root folder on hdfs and the records in YARN Service Registry.
```
DELETE http://localhost:9191/ws/v1/services/${SERVICE_NAME}
DELETE http://localhost:8088/ws/v1/services/${SERVICE_NAME}
```
## Services UI with YARN UI2 and Timeline Service v2

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
View File

@ -37,7 +37,7 @@ License URL: http://www.apache.org/licenses/LICENSE-2.0.html
### URI scheme
Host: host.mycompany.com
Port: 9191(default)
Port: 8088(default RM port)
Schemes: HTTP
@ -368,7 +368,7 @@ The current status of a submitted service, returned as a response to the GET API
## Examples
### Create a simple single-component service with most attribute values as defaults
POST URL - http://localhost:9191/ws/v1/services
POST URL - http://localhost:8088/ws/v1/services
##### POST Request JSON
```json
@ -394,7 +394,7 @@ POST URL - http://localhost:9191/ws/v1/services
```
##### GET Response JSON
GET URL - http://localhost:9191/ws/v1/services/hello-world
GET URL - http://localhost:8088/ws/v1/services/hello-world
Note, lifetime value of -1 means unlimited lifetime.
@ -452,7 +452,7 @@ Note, lifetime value of -1 means unlimited lifetime.
```
### Update to modify the lifetime of a service
PUT URL - http://localhost:9191/ws/v1/services/hello-world
PUT URL - http://localhost:8088/ws/v1/services/hello-world
##### PUT Request JSON
@ -464,7 +464,7 @@ Note, irrespective of what the current lifetime value is, this update request wi
}
```
### Stop a service
PUT URL - http://localhost:9191/ws/v1/services/hello-world
PUT URL - http://localhost:8088/ws/v1/services/hello-world
##### PUT Request JSON
```json
@ -474,7 +474,7 @@ PUT URL - http://localhost:9191/ws/v1/services/hello-world
```
### Start a service
PUT URL - http://localhost:9191/ws/v1/services/hello-world
PUT URL - http://localhost:8088/ws/v1/services/hello-world
##### PUT Request JSON
```json
@ -484,7 +484,7 @@ PUT URL - http://localhost:9191/ws/v1/services/hello-world
```
### Update to flex up/down the no of containers (instances) of a component of a service
PUT URL - http://localhost:9191/ws/v1/services/hello-world/components/hello
PUT URL - http://localhost:8088/ws/v1/services/hello-world/components/hello
##### PUT Request JSON
```json
@ -495,12 +495,12 @@ PUT URL - http://localhost:9191/ws/v1/services/hello-world/components/hello
```
### Destroy a service
DELETE URL - http://localhost:9191/ws/v1/services/hello-world
DELETE URL - http://localhost:8088/ws/v1/services/hello-world
***
### Create a complicated service - HBase
POST URL - http://localhost:9191:/ws/v1/services/hbase-app-1
POST URL - http://localhost:8088:/ws/v1/services/hbase-app-1
##### POST Request JSON
@ -595,4 +595,4 @@ POST URL - http://localhost:9191:/ws/v1/services/hbase-app-1
"Proxied HBase Master Status UI": "http://app-proxy/${DOMAIN}/${USER}/${SERVICE_NAME}/hbasemaster/16010/"
}
}
```
```