big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-5770. Performance improvement of native-services REST API service. Contributed by Gour Saha

Browse Source
This commit is contained in:
Billie Rinaldi 2016-10-26 08:34:39 -07:00 committed by Jian He
parent 1d7a42e795
commit ef5a3628c2
6 changed files with 107 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

144
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
View File

@ -20,8 +20,7 @@
import static org.apache.hadoop.yarn.services.utils.RestApiConstants.*;
import static org.apache.hadoop.yarn.services.utils.RestApiErrorMessages.*;
import java.io.File;
import java.io.FileReader;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.reflect.UndeclaredThrowableException;
import java.security.PrivilegedExceptionAction;
@ -36,7 +35,6 @@
import java.util.Set;
import java.util.regex.Pattern;
import javax.inject.Singleton;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.GET;
@ -52,6 +50,7 @@
import org.apache.commons.lang.SerializationUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.PathNotFoundException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@ -78,7 +77,6 @@
import org.apache.slider.common.params.ActionFreezeArgs;
import org.apache.slider.common.params.ActionListArgs;
import org.apache.slider.common.params.ActionRegistryArgs;
import org.apache.slider.common.params.ActionStatusArgs;
import org.apache.slider.common.params.ActionThawArgs;
import org.apache.slider.common.params.ComponentArgsDelegate;
import org.apache.slider.common.tools.SliderUtils;
@ -98,6 +96,7 @@
import com.google.gson.JsonNull;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.inject.Singleton;
@Singleton
@Path(APPLICATIONS_API_RESOURCE_PATH)
@ -109,6 +108,11 @@ public class ApplicationApiService implements ApplicationApi {
private static org.apache.hadoop.conf.Configuration SLIDER_CONFIG;
private static UserGroupInformation SLIDER_USER;
private static SliderClient SLIDER_CLIENT;
private static Response SLIDER_VERSION;
private static final JsonParser JSON_PARSER = new JsonParser();
private static final JsonObject EMPTY_JSON_OBJECT = new JsonObject();
private static final ActionListArgs ACTION_LIST_ARGS = new ActionListArgs();
private static final ActionFreezeArgs ACTION_FREEZE_ARGS = new ActionFreezeArgs();
static {
init();
@ -119,24 +123,27 @@ protected static void init() {
SLIDER_CONFIG = getSliderClientConfiguration();
SLIDER_USER = getSliderUser();
SLIDER_CLIENT = createSliderClient();
SLIDER_VERSION = initSliderVersion();
}
@GET
@Path("/slider-version")
@Path("/versions/slider-version")
@Consumes({ MediaType.APPLICATION_JSON })
@Produces({ MediaType.APPLICATION_JSON })
public Response getSliderVersion() {
logger.info("GET: getSliderVersion");
return SLIDER_VERSION;
}
private static Response initSliderVersion() {
Map<String, Object> metadata = new HashMap<>();
BuildHelper.addBuildMetadata(metadata, "org.apache.hadoop.yarn.services");
String sliderVersion = metadata.toString();
logger.info("Slider version = {}", sliderVersion);
String hadoopVersion = SliderVersionInfo.getHadoopVersionString();
logger.info("Hadoop version = {}", hadoopVersion);
return Response.ok(
"{ \"slider_version\": \"" + sliderVersion
+ "\", \"hadoop_version\": \"" + hadoopVersion + "\"}").build();
return Response.ok("{ \"slider_version\": \"" + sliderVersion
+ "\", \"hadoop_version\": \"" + hadoopVersion + "\"}").build();
}
@POST
@ -196,7 +203,8 @@ protected void validateApplicationPostPayload(Application application,
}
// If the application has no components do top-level checks
if (application.getComponents() == null) {
if (application.getComponents() == null
|| application.getComponents().size() == 0) {
// artifact
if (application.getArtifact() == null) {
throw new IllegalArgumentException(ERROR_ARTIFACT_INVALID);
@ -222,6 +230,9 @@ protected void validateApplicationPostPayload(Application application,
if (application.getNumberOfContainers() == null) {
throw new IllegalArgumentException(ERROR_CONTAINERS_COUNT_INVALID);
}
// Since it is a simple app with no components, create a default component
application.setComponents(getDefaultComponentAsList(application));
} else {
// If the application has components, then run checks for each component.
// Let global values take effect if component level values are not
@ -274,11 +285,6 @@ protected void validateApplicationPostPayload(Application application,
}
}
// If it is a simple app with no components, then create a default component
if (application.getComponents() == null) {
application.setComponents(getDefaultComponentAsList(application));
}
// Application lifetime if not specified, is set to unlimited lifetime
if (application.getLifetime() == null) {
application.setLifetime(DEFAULT_UNLIMITED_LIFETIME);
@ -853,15 +859,12 @@ public Response getApplication(@PathParam("app_name") String appName) {
// TODO: add status
app.setState(ApplicationState.ACCEPTED);
JsonObject appStatus = null;
JsonObject appRegistryDocker = null;
JsonObject appRegistryQuicklinks = null;
try {
appStatus = getSliderApplicationStatus(appName);
appRegistryDocker = getSliderApplicationRegistry(appName, "docker");
appRegistryQuicklinks = getSliderApplicationRegistry(appName,
"quicklinks");
return populateAppData(app, appStatus, appRegistryDocker,
appRegistryQuicklinks);
return populateAppData(app, appStatus, appRegistryQuicklinks);
} catch (BadClusterStateException | NotFoundException e) {
logger.error(
"Get application failed, application not in running state yet", e);
@ -881,7 +884,7 @@ public Response getApplication(@PathParam("app_name") String appName) {
}
private Response populateAppData(Application app, JsonObject appStatus,
JsonObject appRegistryDocker, JsonObject appRegistryQuicklinks) {
JsonObject appRegistryQuicklinks) {
String appName = jsonGetAsString(appStatus, "name");
Long totalNumberOfRunningContainers = 0L;
Long totalExpectedNumberOfRunningContainers = 0L;
@ -944,7 +947,7 @@ private Response populateAppData(Application app, JsonObject appStatus,
JsonObject applicationStatistics = jsonGetAsObject(appStatus, "statistics");
if (applicationRoles == null) {
// initialize to empty object to avoid too many null checks
applicationRoles = new JsonObject();
applicationRoles = EMPTY_JSON_OBJECT;
}
if (applicationStatus != null) {
JsonObject applicationLive = jsonGetAsObject(applicationStatus, "live");
@ -954,8 +957,9 @@ private Response populateAppData(Application app, JsonObject appStatus,
continue;
}
componentNames.add(entry.getKey());
JsonObject componentRole = applicationRoles.get(entry.getKey()) == null ? new JsonObject()
: applicationRoles.get(entry.getKey()).getAsJsonObject();
JsonObject componentRole = applicationRoles
.get(entry.getKey()) == null ? EMPTY_JSON_OBJECT
: applicationRoles.get(entry.getKey()).getAsJsonObject();
JsonObject liveContainers = entry.getValue().getAsJsonObject();
if (liveContainers != null) {
for (Map.Entry<String, JsonElement> liveContainerEntry : liveContainers
@ -1052,67 +1056,57 @@ private JsonObject jsonGetAsObject(JsonObject object, String key) {
private JsonObject getSliderApplicationStatus(final String appName)
throws IOException, YarnException, InterruptedException {
final File appStatusOutputFile = File.createTempFile("status_", ".json");
final ActionStatusArgs statusArgs = new ActionStatusArgs();
statusArgs.output = appStatusOutputFile.getAbsolutePath();
return invokeSliderClientRunnable(new SliderClientContextRunnable<JsonObject>() {
@Override
public JsonObject run(SliderClient sliderClient) throws YarnException,
IOException, InterruptedException {
sliderClient.actionStatus(appName, statusArgs);
JsonParser parser = new JsonParser();
FileReader reader = null;
JsonElement statusElement = null;
try {
reader = new FileReader(appStatusOutputFile);
statusElement = parser.parse(reader);
} finally {
if (reader != null) {
reader.close();
return invokeSliderClientRunnable(
new SliderClientContextRunnable<JsonObject>() {
@Override
public JsonObject run(SliderClient sliderClient)
throws YarnException, IOException, InterruptedException {
String status = null;
try {
status = sliderClient.actionStatus(appName);
} catch (Exception e) {
logger.error("Exception calling slider.actionStatus", e);
return EMPTY_JSON_OBJECT;
}
JsonElement statusElement = JSON_PARSER.parse(status);
return (statusElement == null || statusElement instanceof JsonNull)
? EMPTY_JSON_OBJECT : (JsonObject) statusElement;
}
appStatusOutputFile.delete();
}
return (statusElement == null || statusElement instanceof JsonNull) ?
new JsonObject() : (JsonObject) statusElement;
}
});
});
}
private JsonObject getSliderApplicationRegistry(final String appName,
final String registryName) throws IOException, YarnException,
InterruptedException {
final File appRegistryOutputFile = File
.createTempFile("registry_", ".json");
final String registryName)
throws IOException, YarnException, InterruptedException {
final ActionRegistryArgs registryArgs = new ActionRegistryArgs();
registryArgs.out = appRegistryOutputFile;
registryArgs.name = appName;
registryArgs.getConf = registryName;
registryArgs.format = ConfigFormat.JSON.toString();
return invokeSliderClientRunnable(new SliderClientContextRunnable<JsonObject>() {
@Override
public JsonObject run(SliderClient sliderClient) throws YarnException,
IOException, InterruptedException {
sliderClient.actionRegistry(registryArgs);
JsonParser parser = new JsonParser();
FileReader reader = null;
JsonElement registryElement = null;
try {
reader = new FileReader(appRegistryOutputFile);
registryElement = parser.parse(reader);
} catch (Throwable t) {
logger.error("Error reading file {}", appRegistryOutputFile);
} finally {
if (reader != null) {
reader.close();
return invokeSliderClientRunnable(
new SliderClientContextRunnable<JsonObject>() {
@Override
public JsonObject run(SliderClient sliderClient)
throws YarnException, IOException, InterruptedException {
String registry = null;
try {
registry = sliderClient.actionRegistryGetConfig(registryArgs)
.asJson();
} catch (FileNotFoundException | PathNotFoundException e) {
// ignore and return empty object
return EMPTY_JSON_OBJECT;
} catch (Exception e) {
logger.error("Exception calling slider.actionRegistryGetConfig",
e);
return EMPTY_JSON_OBJECT;
}
JsonElement registryElement = JSON_PARSER.parse(registry);
return (registryElement == null
|| registryElement instanceof JsonNull) ? EMPTY_JSON_OBJECT
: (JsonObject) registryElement;
}
appRegistryOutputFile.delete();
}
return (registryElement == null || registryElement instanceof JsonNull) ?
new JsonObject() : (JsonObject) registryElement;
}
});
});
}
private Integer getSliderList(final String appName)
@ -1130,8 +1124,7 @@ public Integer run(SliderClient sliderClient) throws YarnException,
if (liveOnly) {
status = sliderClient.actionList(appName);
} else {
ActionListArgs listArgs = new ActionListArgs();
status = sliderClient.actionList(appName, listArgs);
status = sliderClient.actionList(appName, ACTION_LIST_ARGS);
}
return status;
}
@ -1228,8 +1221,7 @@ private Response stopSliderApplication(final String appName)
@Override
public Response run(SliderClient sliderClient) throws YarnException,
IOException, InterruptedException {
ActionFreezeArgs freezeArgs = new ActionFreezeArgs();
int returnCode = sliderClient.actionFreeze(appName, freezeArgs);
int returnCode = sliderClient.actionFreeze(appName, ACTION_FREEZE_ARGS);
if (returnCode == 0) {
logger.info("Successfully stopped application {}", appName);
return Response.status(Status.NO_CONTENT).build();

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
View File

@ -44,7 +44,8 @@
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
@XmlRootElement
@JsonInclude(JsonInclude.Include.NON_NULL)
@JsonPropertyOrder({ " name, state, resource, numberOfContainers, lifetime, containers " })
@JsonPropertyOrder({ "name", "state", "resource", "number_of_containers",
"lifetime", "containers" })
public class Application extends BaseResource {
private static final long serialVersionUID = -4491694636566094885L;
@ -174,8 +175,8 @@ public Application launchTime(Date launchTime) {
@ApiModelProperty(example = "null", value = "The time when the application was created, e.g. 2016-03-16T01:01:49.000Z.")
@JsonProperty("launch_time")
public String getLaunchTime() {
return launchTime == null ? null : launchTime.toString();
public Date getLaunchTime() {
return launchTime == null ? null : (Date) launchTime.clone();
}
@XmlElement(name = "launch_time")

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Container.java
View File

@ -79,8 +79,8 @@ public Container launchTime(Date launchTime) {
@ApiModelProperty(example = "null", value = "The time when the container was created, e.g. 2016-03-16T01:01:49.000Z. This will most likely be different from cluster launch time.")
@JsonProperty("launch_time")
public String getLaunchTime() {
return launchTime == null ? null : launchTime.toString();
public Date getLaunchTime() {
return launchTime == null ? null : (Date) launchTime.clone();
}
@XmlElement(name = "launch_time")

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java
View File

@ -27,6 +27,7 @@
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.services.api.impl.ApplicationApiService;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
import org.mortbay.jetty.webapp.Configuration;
@ -95,13 +96,10 @@ protected void startWebApp() throws IOException {
.setName("services-rest-api")
.addEndpoint(URI.create("http://" + webHost + ":" + webPort)).build();
String apiPackages = "org.apache.hadoop.yarn.services.api" + SEP
+ "org.apache.hadoop.yarn.services.api.impl" + SEP
+ "org.apache.hadoop.yarn.services.resource" + SEP
+ "org.apache.hadoop.yarn.services.utils" + SEP
+ "org.apache.hadoop.yarn.services.webapp" + SEP
+ GenericExceptionHandler.class.getPackage().getName() + SEP
+ YarnJacksonJaxbJsonProvider.class.getPackage().getName();
String apiPackages =
ApplicationApiService.class.getPackage().getName() + SEP
+ GenericExceptionHandler.class.getPackage().getName() + SEP
+ YarnJacksonJaxbJsonProvider.class.getPackage().getName();
applicationApiServer.addJerseyResourcePackage(apiPackages, CONTEXT_ROOT
+ "/*");

25
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
View File

@ -3137,22 +3137,31 @@ private SliderClusterProtocol connect(ApplicationReport app)
@Override
@VisibleForTesting
public int actionStatus(String clustername, ActionStatusArgs statusArgs) throws
YarnException,
IOException {
verifyBindingsDefined();
validateClusterName(clustername);
public int actionStatus(String clustername, ActionStatusArgs statusArgs)
throws YarnException, IOException {
ClusterDescription status = verifyAndGetClusterDescription(clustername);
String outfile = statusArgs.getOutput();
ClusterDescription status = getClusterDescription(clustername);
String text = status.toJsonString();
if (outfile == null) {
log.info(text);
log.info(status.toJsonString());
} else {
status.save(new File(outfile).getAbsoluteFile());
}
return EXIT_SUCCESS;
}
@Override
public String actionStatus(String clustername)
throws YarnException, IOException {
return verifyAndGetClusterDescription(clustername).toJsonString();
}
private ClusterDescription verifyAndGetClusterDescription(String clustername)
throws YarnException, IOException {
verifyBindingsDefined();
validateClusterName(clustername);
return getClusterDescription(clustername);
}
@Override
public int actionVersion() {
SliderVersionInfo.loadAndPrintVersionInfo(log);

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
View File

@ -279,6 +279,17 @@ String actionEcho(String name, ActionEchoArgs args)
int actionStatus(String clustername, ActionStatusArgs statusArgs)
throws YarnException, IOException;
/**
* Status operation which returns the status object as a string instead of
* printing it to the console or file.
*
* @param clustername cluster name
* @return cluster status details
* @throws YarnException
* @throws IOException
*/
String actionStatus(String clustername) throws YarnException, IOException;
/**
* Version Details
* @return exit code