big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-6626. Embed REST API service into RM. Contributed by Eric Yang

Browse Source
This commit is contained in:
Jian He 2017-09-28 16:29:22 -07:00
parent 8851209788
commit 9e677fa05c
9 changed files with 89 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -336,6 +336,8 @@ private static void addDeprecatedKeys() {
public static final String YARN_WEBAPP_UI2_WARFILE_PATH = "yarn."
+ "webapp.ui2.war-file-path";
public static final String YARN_API_SERVICES_ENABLE = "yarn."
+ "webapp.api-service.enable";
public static final String RM_RESOURCE_TRACKER_ADDRESS =
RM_PREFIX + "resource-tracker.address";

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
View File

@ -17,6 +17,7 @@
package org.apache.hadoop.yarn.service.webapp;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.VersionInfo;
@ -57,6 +58,12 @@
@Singleton
@Path(CONTEXT_ROOT)
public class ApiServer {
@Inject
public ApiServer(Configuration conf) {
super();
}
private static final Logger LOG =
LoggerFactory.getLogger(ApiServer.class);
private static Configuration YARN_CONFIG = new YarnConfiguration();

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
View File

@ -244,12 +244,6 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-resourcemanager</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Artifact.java
View File

@ -23,6 +23,9 @@
import java.io.Serializable;
import java.util.Objects;
import javax.xml.bind.annotation.XmlEnum;
import javax.xml.bind.annotation.XmlType;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonValue;
@ -42,6 +45,11 @@ public class Artifact implements Serializable {
private String id = null;
/**
* Artifact Type. DOCKER, TARBALL or SERVICE
**/
@XmlType(name = "artifact_type")
@XmlEnum
public enum TypeEnum {
DOCKER("DOCKER"), TARBALL("TARBALL"), SERVICE("SERVICE");

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFile.java
View File

@ -26,7 +26,10 @@
import org.apache.hadoop.classification.InterfaceStability;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlEnum;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlType;
import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
@ -45,6 +48,11 @@
public class ConfigFile implements Serializable {
private static final long serialVersionUID = -7009402089417704612L;
/**
* Config Type. XML, JSON, YAML, TEMPLATE, ENV and HADOOP_XML are supported.
**/
@XmlType(name = "config_type")
@XmlEnum
public enum TypeEnum {
XML("XML"), PROPERTIES("PROPERTIES"), JSON("JSON"), YAML("YAML"), TEMPLATE(
"TEMPLATE"), ENV("ENV"), HADOOP_XML("HADOOP_XML"),;

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
View File

@ -25,6 +25,9 @@
import java.util.Map;
import java.util.Objects;
import javax.xml.bind.annotation.XmlEnum;
import javax.xml.bind.annotation.XmlType;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonValue;
import org.apache.hadoop.classification.InterfaceAudience;
@ -43,6 +46,11 @@
public class ReadinessCheck implements Serializable {
private static final long serialVersionUID = -3836839816887186801L;
/**
* Type. HTTP and PORT
**/
@XmlType(name = "type")
@XmlEnum
public enum TypeEnum {
HTTP("HTTP"),
PORT("PORT");

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -211,6 +211,14 @@
<value></value>
</property>
<property>
<description>
Enable services rest api on ResourceManager.
</description>
<name>yarn.webapp.api-service.enable</name>
<value>false</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>${yarn.resourcemanager.hostname}:8031</value>

19
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
View File

@ -22,6 +22,8 @@
import java.net.InetSocketAddress;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.RMHAUtils;
@ -38,8 +40,12 @@
*/
public class RMWebApp extends WebApp implements YarnWebParams {
private static final Log LOG =
LogFactory.getLog(RMWebApp.class.getName());
private final ResourceManager rm;
private boolean standby = false;
private final static String APISERVER =
"org.apache.hadoop.yarn.service.webapp.ApiServer";
public RMWebApp(ResourceManager rm) {
this.rm = rm;
@ -53,6 +59,19 @@ public void setup() {
bind(RMWebApp.class).toInstance(this);
if (rm != null) {
boolean enableServiceApi = rm.getConfig()
.getBoolean(YarnConfiguration.YARN_API_SERVICES_ENABLE, false);
if (enableServiceApi) {
try {
// Use reflection here to load ApiServer class,
// this is done to avoid creating cyclic dependency
// between maven projects.
Class<?> apiServer = Class.forName(APISERVER);
bind(apiServer);
} catch (ClassNotFoundException e) {
LOG.warn("ApiServer REST API is not activated.");
}
}
bind(ResourceManager.class).toInstance(rm);
}
route("/", RmController.class);

34
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
View File

@ -101,17 +101,41 @@ yarn service destroy ${SERVICE_NAME}
```
## Manage services on YARN via REST API
Below steps walk you through deploying services on YARN via REST API.
Refer to [API doc](YarnServiceAPI.md) for the detailed API specificatiosn.
### Start API-Server for deploying services on YARN
API server is the service that sits in front of YARN ResourceManager and lets users submit their API specs via HTTP.
YARN API Server REST API can be activated in two modes: embedded or standalone.
### Start Embedded API-Server as part of ResourceManager
For running inside ResourceManager, add this property to `yarn-site.xml` and restart ResourceManager.
```
<property>
<description>
Enable services rest api on ResourceManager.
</description>
<name>yarn.webapp.api-service.enable</name>
<value>true</value>
</property>
```
Services can be deployed on YARN through the ResourceManager web endpoint.
### Start Standalone API-Server for deploying services on YARN
API server is the service that sits in front of YARN ResourceManager and lets users submit their service specs via HTTP.
```
yarn --daemon start apiserver
```
The above command starts the API Server on the localhost at port 9191 by default.
Refer to [API doc](YarnServiceAPI.md) for the detailed API specificatiosn.
### Deploy a service
POST the aforementioned example service definition to the api-server endpoint:
POST the aforementioned example service definition to the ResourceManager api-server endpoint:
```
POST http://localhost:8088/ws/v1/services
```
Or standalone API server:
```
POST http://localhost:9191/ws/v1/services
```