big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-7186. Add examples in yarn-service. Contributed by Jian He

Browse Source
This commit is contained in:
Billie Rinaldi 2017-09-14 15:43:00 -07:00 committed by Jian He
parent 307d55b3e1
commit 37c9b7327d
14 changed files with 91 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
View File

@ -97,6 +97,13 @@
<directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/conf</directory>
<outputDirectory>etc/hadoop</outputDirectory>
</fileSet>
<fileSet>
<directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/yarn-service-examples</outputDirectory>
<includes>
<include>**/*</include>
</includes>
</fileSet>
<fileSet>
<directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/sleeper/sleeper.json Normal file
View File

@ -0,0 +1,15 @@
{
"name": "sleeper-service",
"components" :
[
{
"name": "sleeper",
"number_of_containers": 2,
"launch_command": "sleep 900000",
"resource": {
"cpus": 1,
"memory": "256"
}
}
]
}

26
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
View File

@ -66,6 +66,7 @@
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.api.records.ServiceState;
import org.apache.hadoop.yarn.service.client.params.AbstractClusterBuildingActionArgs;
import org.apache.hadoop.yarn.service.client.params.ActionCreateArgs;
import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs;
import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs;
import org.apache.hadoop.yarn.service.client.params.Arguments;
@ -160,12 +161,15 @@ private Service loadAppJsonFromLocalFS(
AbstractClusterBuildingActionArgs args) throws IOException {
File file = args.getFile();
Path filePath = new Path(file.getAbsolutePath());
LOG.info("Loading app json from: " + filePath);
LOG.info("Loading service definition from: " + filePath);
Service service = jsonSerDeser
.load(FileSystem.getLocal(getConfig()), filePath);
if (args.lifetime > 0) {
service.setLifetime(args.lifetime);
}
if (!StringUtils.isEmpty(args.getServiceName())) {
service.setName(args.getServiceName());
}
return service;
}
@ -182,9 +186,23 @@ public int actionBuild(Service service)
return EXIT_SUCCESS;
}
public int actionCreate(AbstractClusterBuildingActionArgs args)
public int actionCreate(ActionCreateArgs args)
throws IOException, YarnException {
actionCreate(loadAppJsonFromLocalFS(args));
Service serviceDef;
if (args.file != null) {
serviceDef = loadAppJsonFromLocalFS(args);
} else if (!StringUtils.isEmpty(args.example)) {
// create an example service
String yarnHome = System
.getenv(ApplicationConstants.Environment.HADOOP_YARN_HOME.key());
args.file = new File(MessageFormat
.format("{0}/share/hadoop/yarn/yarn-service-examples/{1}/{2}.json",
yarnHome, args.example, args.example));
serviceDef = loadAppJsonFromLocalFS(args);
} else {
throw new YarnException("No service definition provided!");
}
actionCreate(serviceDef);
return EXIT_SUCCESS;
}
@ -213,7 +231,7 @@ protected int actionFlexByCLI(ClientArgs args)
Map<String, Long> componentCounts =
new HashMap<>(flexArgs.getComponentMap().size());
Service persistedService =
ServiceApiUtil.loadService(fs, flexArgs.getClusterName());
ServiceApiUtil.loadService(fs, flexArgs.getServiceName());
if (!StringUtils.isEmpty(persistedService.getId())) {
cachedAppIds.put(persistedService.getName(),
ApplicationId.fromString(persistedService.getId()));

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractActionArgs.java
View File

@ -50,7 +50,7 @@ protected AbstractActionArgs() {
* get the name: relies on arg 1 being the cluster name in all operations
* @return the name argument, null if there is none
*/
public String getClusterName() {
public String getServiceName() {
return (parameters.isEmpty()) ? null : parameters.get(0);
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
View File

@ -28,7 +28,7 @@
*/
public abstract class AbstractClusterBuildingActionArgs
extends AbstractActionArgs {
@Parameter(names = { ARG_FILE, ARG_FILE_SHORT }, required = true,
@Parameter(names = { ARG_FILE, ARG_FILE_SHORT },
description = "The path to the service definition file in JSON format.")
public File file;

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionBuildArgs.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameters;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
@Parameters(commandNames = { SliderActions.ACTION_BUILD},
commandDescription = SliderActions.DESCRIBE_ACTION_BUILD)
@ -30,7 +31,9 @@ public String getActionName() {
}
@Override
public int getMinParams() {
return 0;
public void validate() throws BadCommandArgumentsException {
if (file == null) {
throw new BadCommandArgumentsException("No service definition provided.");
}
}
}

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionCreateArgs.java
View File

@ -18,21 +18,29 @@
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
@Parameters(commandNames = { SliderActions.ACTION_CREATE},
commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
public class ActionCreateArgs extends AbstractClusterBuildingActionArgs {
@Parameter(names = { ARG_EXAMPLE, ARG_EXAMPLE_SHORT },
description = "The name of the example service such as sleeper")
public String example;
@Override
public String getActionName() {
return SliderActions.ACTION_CREATE;
}
@Override
public int getMinParams() {
return 0;
public void validate() throws BadCommandArgumentsException {
if (file == null && example == null) {
throw new BadCommandArgumentsException("No service definition provided.");
}
}
}

26
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionExamples.java
View File

@ -1,26 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.service.client.params;
import com.beust.jcommander.Parameters;
@Parameters(commandNames = { SliderActions.ACTION_EXAMPLES},
commandDescription = SliderActions.DESCRIBE_ACTION_EXAMPLES)
public class ActionExamples {
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
View File

@ -42,8 +42,8 @@ public interface Arguments {
String ARG_DELETE = "--delete";
String ARG_DEST = "--dest";
String ARG_DESTDIR = "--destdir";
String ARG_FILESYSTEM = "--fs";
String ARG_FILESYSTEM_LONG = "--filesystem";
String ARG_EXAMPLE = "--example";
String ARG_EXAMPLE_SHORT = "-e";
String ARG_FOLDER = "--folder";
String ARG_FORCE = "--force";
String ARG_FORMAT = "--format";

44
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ClientArgs.java
View File

@ -19,8 +19,6 @@
package org.apache.hadoop.yarn.service.client.params;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
import org.apache.hadoop.yarn.service.utils.SliderUtils;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
@ -104,51 +102,10 @@ public ActionDependencyArgs getActionDependencyArgs() {
return actionDependencyArgs;
}
public ActionDestroyArgs getActionDestroyArgs() {
return actionDestroyArgs;
}
public ActionExistsArgs getActionExistsArgs() {
return actionExistsArgs;
}
public ActionFlexArgs getActionFlexArgs() {
return actionFlexArgs;
}
public ActionFreezeArgs getActionFreezeArgs() {
return actionFreezeArgs;
}
public ActionListArgs getActionListArgs() {
return actionListArgs;
}
public ActionRegistryArgs getActionRegistryArgs() {
return actionRegistryArgs;
}
public ActionResolveArgs getActionResolveArgs() {
return actionResolveArgs;
}
public ActionResourceArgs getActionResourceArgs() {
return actionResourceArgs;
}
public ActionStatusArgs getActionStatusArgs() {
return actionStatusArgs;
}
public ActionThawArgs getActionThawArgs() {
return actionThawArgs;
}
public ActionTokensArgs getActionTokenArgs() {
return actionTokenArgs;
}
/**
* Look at the chosen action and bind it as the core action for the operation.
* @throws SliderException bad argument or similar
@ -227,7 +184,6 @@ public void applyAction() throws SliderException {
case ACTION_UPDATE:
bindCoreAction(actionUpdateArgs);
break;
default:
throw new BadCommandArgumentsException(ErrorStrings.ERROR_UNKNOWN_ACTION
+ " " + action);

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/CommonArgs.java
View File

@ -24,7 +24,6 @@
import com.beust.jcommander.ParameterException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.service.utils.SliderUtils;
import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
@ -88,7 +87,7 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
* @return the name argument, null if there is none
*/
public String getClusterName() {
return coreAction.getClusterName();
return coreAction.getServiceName();
}
protected CommonArgs(String[] args) {
@ -142,10 +141,6 @@ public static String usage(CommonArgs serviceArgs, String commandOfInterest) {
return result;
}
public static String usage(CommonArgs serviceArgs) {
return usage(serviceArgs, null);
}
/**
* Parse routine -includes registering the action-specific argument classes
* and postprocess it
@ -164,14 +159,6 @@ public void parse() throws SliderException {
postProcess();
}
/**
* Add a command
* @param name action
* @param arg value
*/
protected void addAction(String name, Object arg) {
commander.addCommand(name, arg);
}
protected void addActions(Object... actions) {
for (Object action : actions) {

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/SliderActions.java
View File

@ -30,7 +30,6 @@ public interface SliderActions {
String ACTION_UPGRADE = "upgrade";
String ACTION_DESTROY = "destroy";
String ACTION_EXISTS = "exists";
String ACTION_EXAMPLES = "examples";
String ACTION_FLEX = "flex";
String ACTION_STOP = "stop";
String ACTION_HELP = "help";
@ -59,7 +58,6 @@ public interface SliderActions {
"Destroy a stopped service, service must be stopped first before destroying.";
String DESCRIBE_ACTION_EXISTS =
"Probe for a service running";
String DESCRIBE_ACTION_EXAMPLES = "Run an example service on YARN";
String DESCRIBE_ACTION_FLEX = "Flex a service's component by increasing or decreasing the number of containers.";
String DESCRIBE_ACTION_FREEZE =
"Stop a running service";

34
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
View File

@ -76,28 +76,32 @@ Usage `yarn service [sub-command] [service-name] [options]`
* `build`: Build a service with its specifications, but do not start it.
```
Usage: yarn service build --file [file]
```
Usage: yarn service build [service-name] --file [file]
Fields:
service-name Optional. If specified, it will override the name in the service definition.
| COMMAND\_OPTIONS | Description |
|:---- |:---- |
| --file or -f | The local path to the service definition file |
Options:
--file,-f The local path to the service definition file
```
* `create`: create a service, it's equivalent to first invoke build and then start.
```
Usage: yarn service create --file [file]
Usage: yarn service create [service-name] --file [file]
Fields:
service-name Optional. If specified, it will override the name in the service definition.
Options:
--file,-f The local path to the service definition file.
--example,-e The name of the example service such as:
Sleeper A simple service that launches a few non-docker sleep containers on YARN.
```
| COMMAND\_OPTIONS | Description |
|:---- |:---- |
| --file or -f | The local path to the service definition file |
* `dependency`: Yarn service framework dependency (libraries) management.
```
Usage: yarn service dependency [options]
Option:
--upload Pre-upload the dependency jars onto HDFS to expediate service launch process.
```
| COMMAND\_OPTIONS | Description |
|:---- |:---- |
| --upload | Pre-upload the dependency jars onto HDFS to expediate service launch process. |
* `destroy`: Destroy a stopped service, service must be stopped first before destroying.
```
@ -106,10 +110,10 @@ Usage `yarn service [sub-command] [service-name] [options]`
* `flex`: Flex a service's component by increasing or decreasing the number of containers.
```
Usage: yarn service flex [service-name] --component [component-name] [count]
Options:
--component [component-name] [count]
Specifies the component name and its number of containers. e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3.
```
| COMMAND\_OPTIONS | Description |
|:---- |:---- |
| --component [component-name] [count] | Specifies the component name and its number of containers. e.g. +1 incr by 1, -2 decr by 2, and 3 makes final count 3.|
* `status`: Get the status of a service.
```
Usage: yarn service status [service-name]

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
View File

@ -43,7 +43,14 @@ Below is a simple service definition that launches sleep containers on YARN by w
]
}
```
User can simply run a pre-built example service on YARN using below command:
```
yarn service create [service-name] --example [example-name]
```
e.g. Below command launches a `sleeper` service named as `my-sleeper` on YARN.
```
yarn service create my-sleeper --example sleeper
```
For launching docker based services using YARN Service framework, please refer to [API doc](YarnServiceAPI.md).
## Manage services on YARN via CLI