From ef5a3628c2428a1f8b0b17e1ff2aabe6820b63d5 Mon Sep 17 00:00:00 2001 From: Billie Rinaldi Date: Wed, 26 Oct 2016 08:34:39 -0700 Subject: [PATCH] YARN-5770. Performance improvement of native-services REST API service. Contributed by Gour Saha --- .../api/impl/ApplicationApiService.java | 144 +++++++++--------- .../yarn/services/resource/Application.java | 7 +- .../yarn/services/resource/Container.java | 4 +- .../services/webapp/ApplicationApiWebApp.java | 12 +- .../apache/slider/client/SliderClient.java | 25 ++- .../apache/slider/client/SliderClientAPI.java | 11 ++ 6 files changed, 107 insertions(+), 96 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java index 73df4a11a3..cf43ac2531 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java @@ -20,8 +20,7 @@ import static org.apache.hadoop.yarn.services.utils.RestApiConstants.*; import static org.apache.hadoop.yarn.services.utils.RestApiErrorMessages.*; -import java.io.File; -import java.io.FileReader; +import java.io.FileNotFoundException; import java.io.IOException; import java.lang.reflect.UndeclaredThrowableException; import java.security.PrivilegedExceptionAction; @@ -36,7 +35,6 @@ import java.util.Set; import java.util.regex.Pattern; -import javax.inject.Singleton; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.GET; @@ -52,6 +50,7 @@ import org.apache.commons.lang.SerializationUtils; import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.fs.PathNotFoundException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -78,7 +77,6 @@ import org.apache.slider.common.params.ActionFreezeArgs; import org.apache.slider.common.params.ActionListArgs; import org.apache.slider.common.params.ActionRegistryArgs; -import org.apache.slider.common.params.ActionStatusArgs; import org.apache.slider.common.params.ActionThawArgs; import org.apache.slider.common.params.ComponentArgsDelegate; import org.apache.slider.common.tools.SliderUtils; @@ -98,6 +96,7 @@ import com.google.gson.JsonNull; import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import com.google.inject.Singleton; @Singleton @Path(APPLICATIONS_API_RESOURCE_PATH) @@ -109,6 +108,11 @@ public class ApplicationApiService implements ApplicationApi { private static org.apache.hadoop.conf.Configuration SLIDER_CONFIG; private static UserGroupInformation SLIDER_USER; private static SliderClient SLIDER_CLIENT; + private static Response SLIDER_VERSION; + private static final JsonParser JSON_PARSER = new JsonParser(); + private static final JsonObject EMPTY_JSON_OBJECT = new JsonObject(); + private static final ActionListArgs ACTION_LIST_ARGS = new ActionListArgs(); + private static final ActionFreezeArgs ACTION_FREEZE_ARGS = new ActionFreezeArgs(); static { init(); @@ -119,24 +123,27 @@ protected static void init() { SLIDER_CONFIG = getSliderClientConfiguration(); SLIDER_USER = getSliderUser(); SLIDER_CLIENT = createSliderClient(); + SLIDER_VERSION = initSliderVersion(); } @GET - @Path("/slider-version") + @Path("/versions/slider-version") @Consumes({ MediaType.APPLICATION_JSON }) @Produces({ MediaType.APPLICATION_JSON }) public Response getSliderVersion() { logger.info("GET: getSliderVersion"); + return SLIDER_VERSION; + } + private static Response initSliderVersion() { Map metadata = new HashMap<>(); BuildHelper.addBuildMetadata(metadata, "org.apache.hadoop.yarn.services"); String sliderVersion = metadata.toString(); logger.info("Slider version = {}", sliderVersion); String hadoopVersion = SliderVersionInfo.getHadoopVersionString(); logger.info("Hadoop version = {}", hadoopVersion); - return Response.ok( - "{ \"slider_version\": \"" + sliderVersion - + "\", \"hadoop_version\": \"" + hadoopVersion + "\"}").build(); + return Response.ok("{ \"slider_version\": \"" + sliderVersion + + "\", \"hadoop_version\": \"" + hadoopVersion + "\"}").build(); } @POST @@ -196,7 +203,8 @@ protected void validateApplicationPostPayload(Application application, } // If the application has no components do top-level checks - if (application.getComponents() == null) { + if (application.getComponents() == null + || application.getComponents().size() == 0) { // artifact if (application.getArtifact() == null) { throw new IllegalArgumentException(ERROR_ARTIFACT_INVALID); @@ -222,6 +230,9 @@ protected void validateApplicationPostPayload(Application application, if (application.getNumberOfContainers() == null) { throw new IllegalArgumentException(ERROR_CONTAINERS_COUNT_INVALID); } + + // Since it is a simple app with no components, create a default component + application.setComponents(getDefaultComponentAsList(application)); } else { // If the application has components, then run checks for each component. // Let global values take effect if component level values are not @@ -274,11 +285,6 @@ protected void validateApplicationPostPayload(Application application, } } - // If it is a simple app with no components, then create a default component - if (application.getComponents() == null) { - application.setComponents(getDefaultComponentAsList(application)); - } - // Application lifetime if not specified, is set to unlimited lifetime if (application.getLifetime() == null) { application.setLifetime(DEFAULT_UNLIMITED_LIFETIME); @@ -853,15 +859,12 @@ public Response getApplication(@PathParam("app_name") String appName) { // TODO: add status app.setState(ApplicationState.ACCEPTED); JsonObject appStatus = null; - JsonObject appRegistryDocker = null; JsonObject appRegistryQuicklinks = null; try { appStatus = getSliderApplicationStatus(appName); - appRegistryDocker = getSliderApplicationRegistry(appName, "docker"); appRegistryQuicklinks = getSliderApplicationRegistry(appName, "quicklinks"); - return populateAppData(app, appStatus, appRegistryDocker, - appRegistryQuicklinks); + return populateAppData(app, appStatus, appRegistryQuicklinks); } catch (BadClusterStateException | NotFoundException e) { logger.error( "Get application failed, application not in running state yet", e); @@ -881,7 +884,7 @@ public Response getApplication(@PathParam("app_name") String appName) { } private Response populateAppData(Application app, JsonObject appStatus, - JsonObject appRegistryDocker, JsonObject appRegistryQuicklinks) { + JsonObject appRegistryQuicklinks) { String appName = jsonGetAsString(appStatus, "name"); Long totalNumberOfRunningContainers = 0L; Long totalExpectedNumberOfRunningContainers = 0L; @@ -944,7 +947,7 @@ private Response populateAppData(Application app, JsonObject appStatus, JsonObject applicationStatistics = jsonGetAsObject(appStatus, "statistics"); if (applicationRoles == null) { // initialize to empty object to avoid too many null checks - applicationRoles = new JsonObject(); + applicationRoles = EMPTY_JSON_OBJECT; } if (applicationStatus != null) { JsonObject applicationLive = jsonGetAsObject(applicationStatus, "live"); @@ -954,8 +957,9 @@ private Response populateAppData(Application app, JsonObject appStatus, continue; } componentNames.add(entry.getKey()); - JsonObject componentRole = applicationRoles.get(entry.getKey()) == null ? new JsonObject() - : applicationRoles.get(entry.getKey()).getAsJsonObject(); + JsonObject componentRole = applicationRoles + .get(entry.getKey()) == null ? EMPTY_JSON_OBJECT + : applicationRoles.get(entry.getKey()).getAsJsonObject(); JsonObject liveContainers = entry.getValue().getAsJsonObject(); if (liveContainers != null) { for (Map.Entry liveContainerEntry : liveContainers @@ -1052,67 +1056,57 @@ private JsonObject jsonGetAsObject(JsonObject object, String key) { private JsonObject getSliderApplicationStatus(final String appName) throws IOException, YarnException, InterruptedException { - final File appStatusOutputFile = File.createTempFile("status_", ".json"); - final ActionStatusArgs statusArgs = new ActionStatusArgs(); - statusArgs.output = appStatusOutputFile.getAbsolutePath(); - return invokeSliderClientRunnable(new SliderClientContextRunnable() { - @Override - public JsonObject run(SliderClient sliderClient) throws YarnException, - IOException, InterruptedException { - sliderClient.actionStatus(appName, statusArgs); - JsonParser parser = new JsonParser(); - FileReader reader = null; - JsonElement statusElement = null; - try { - reader = new FileReader(appStatusOutputFile); - statusElement = parser.parse(reader); - } finally { - if (reader != null) { - reader.close(); + return invokeSliderClientRunnable( + new SliderClientContextRunnable() { + @Override + public JsonObject run(SliderClient sliderClient) + throws YarnException, IOException, InterruptedException { + String status = null; + try { + status = sliderClient.actionStatus(appName); + } catch (Exception e) { + logger.error("Exception calling slider.actionStatus", e); + return EMPTY_JSON_OBJECT; + } + JsonElement statusElement = JSON_PARSER.parse(status); + return (statusElement == null || statusElement instanceof JsonNull) + ? EMPTY_JSON_OBJECT : (JsonObject) statusElement; } - appStatusOutputFile.delete(); - } - return (statusElement == null || statusElement instanceof JsonNull) ? - new JsonObject() : (JsonObject) statusElement; - } - }); + }); } private JsonObject getSliderApplicationRegistry(final String appName, - final String registryName) throws IOException, YarnException, - InterruptedException { - final File appRegistryOutputFile = File - .createTempFile("registry_", ".json"); + final String registryName) + throws IOException, YarnException, InterruptedException { final ActionRegistryArgs registryArgs = new ActionRegistryArgs(); - registryArgs.out = appRegistryOutputFile; registryArgs.name = appName; registryArgs.getConf = registryName; registryArgs.format = ConfigFormat.JSON.toString(); - return invokeSliderClientRunnable(new SliderClientContextRunnable() { - @Override - public JsonObject run(SliderClient sliderClient) throws YarnException, - IOException, InterruptedException { - sliderClient.actionRegistry(registryArgs); - JsonParser parser = new JsonParser(); - FileReader reader = null; - JsonElement registryElement = null; - try { - reader = new FileReader(appRegistryOutputFile); - registryElement = parser.parse(reader); - } catch (Throwable t) { - logger.error("Error reading file {}", appRegistryOutputFile); - } finally { - if (reader != null) { - reader.close(); + return invokeSliderClientRunnable( + new SliderClientContextRunnable() { + @Override + public JsonObject run(SliderClient sliderClient) + throws YarnException, IOException, InterruptedException { + String registry = null; + try { + registry = sliderClient.actionRegistryGetConfig(registryArgs) + .asJson(); + } catch (FileNotFoundException | PathNotFoundException e) { + // ignore and return empty object + return EMPTY_JSON_OBJECT; + } catch (Exception e) { + logger.error("Exception calling slider.actionRegistryGetConfig", + e); + return EMPTY_JSON_OBJECT; + } + JsonElement registryElement = JSON_PARSER.parse(registry); + return (registryElement == null + || registryElement instanceof JsonNull) ? EMPTY_JSON_OBJECT + : (JsonObject) registryElement; } - appRegistryOutputFile.delete(); - } - return (registryElement == null || registryElement instanceof JsonNull) ? - new JsonObject() : (JsonObject) registryElement; - } - }); + }); } private Integer getSliderList(final String appName) @@ -1130,8 +1124,7 @@ public Integer run(SliderClient sliderClient) throws YarnException, if (liveOnly) { status = sliderClient.actionList(appName); } else { - ActionListArgs listArgs = new ActionListArgs(); - status = sliderClient.actionList(appName, listArgs); + status = sliderClient.actionList(appName, ACTION_LIST_ARGS); } return status; } @@ -1228,8 +1221,7 @@ private Response stopSliderApplication(final String appName) @Override public Response run(SliderClient sliderClient) throws YarnException, IOException, InterruptedException { - ActionFreezeArgs freezeArgs = new ActionFreezeArgs(); - int returnCode = sliderClient.actionFreeze(appName, freezeArgs); + int returnCode = sliderClient.actionFreeze(appName, ACTION_FREEZE_ARGS); if (returnCode == 0) { logger.info("Successfully stopped application {}", appName); return Response.status(Status.NO_CONTENT).build(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java index ed65ad2b22..beeffba3aa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java @@ -44,7 +44,8 @@ @javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00") @XmlRootElement @JsonInclude(JsonInclude.Include.NON_NULL) -@JsonPropertyOrder({ " name, state, resource, numberOfContainers, lifetime, containers " }) +@JsonPropertyOrder({ "name", "state", "resource", "number_of_containers", + "lifetime", "containers" }) public class Application extends BaseResource { private static final long serialVersionUID = -4491694636566094885L; @@ -174,8 +175,8 @@ public Application launchTime(Date launchTime) { @ApiModelProperty(example = "null", value = "The time when the application was created, e.g. 2016-03-16T01:01:49.000Z.") @JsonProperty("launch_time") - public String getLaunchTime() { - return launchTime == null ? null : launchTime.toString(); + public Date getLaunchTime() { + return launchTime == null ? null : (Date) launchTime.clone(); } @XmlElement(name = "launch_time") diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Container.java index a4efdf3c08..f11c7b39a2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Container.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Container.java @@ -79,8 +79,8 @@ public Container launchTime(Date launchTime) { @ApiModelProperty(example = "null", value = "The time when the container was created, e.g. 2016-03-16T01:01:49.000Z. This will most likely be different from cluster launch time.") @JsonProperty("launch_time") - public String getLaunchTime() { - return launchTime == null ? null : launchTime.toString(); + public Date getLaunchTime() { + return launchTime == null ? null : (Date) launchTime.clone(); } @XmlElement(name = "launch_time") diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java index b1b6d7c01f..52a9de6b46 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java @@ -27,6 +27,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.services.api.impl.ApplicationApiService; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; import org.mortbay.jetty.webapp.Configuration; @@ -95,13 +96,10 @@ protected void startWebApp() throws IOException { .setName("services-rest-api") .addEndpoint(URI.create("http://" + webHost + ":" + webPort)).build(); - String apiPackages = "org.apache.hadoop.yarn.services.api" + SEP - + "org.apache.hadoop.yarn.services.api.impl" + SEP - + "org.apache.hadoop.yarn.services.resource" + SEP - + "org.apache.hadoop.yarn.services.utils" + SEP - + "org.apache.hadoop.yarn.services.webapp" + SEP - + GenericExceptionHandler.class.getPackage().getName() + SEP - + YarnJacksonJaxbJsonProvider.class.getPackage().getName(); + String apiPackages = + ApplicationApiService.class.getPackage().getName() + SEP + + GenericExceptionHandler.class.getPackage().getName() + SEP + + YarnJacksonJaxbJsonProvider.class.getPackage().getName(); applicationApiServer.addJerseyResourcePackage(apiPackages, CONTEXT_ROOT + "/*"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 94e51e5a44..4b546cd5b4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -3137,22 +3137,31 @@ private SliderClusterProtocol connect(ApplicationReport app) @Override @VisibleForTesting - public int actionStatus(String clustername, ActionStatusArgs statusArgs) throws - YarnException, - IOException { - verifyBindingsDefined(); - validateClusterName(clustername); + public int actionStatus(String clustername, ActionStatusArgs statusArgs) + throws YarnException, IOException { + ClusterDescription status = verifyAndGetClusterDescription(clustername); String outfile = statusArgs.getOutput(); - ClusterDescription status = getClusterDescription(clustername); - String text = status.toJsonString(); if (outfile == null) { - log.info(text); + log.info(status.toJsonString()); } else { status.save(new File(outfile).getAbsoluteFile()); } return EXIT_SUCCESS; } + @Override + public String actionStatus(String clustername) + throws YarnException, IOException { + return verifyAndGetClusterDescription(clustername).toJsonString(); + } + + private ClusterDescription verifyAndGetClusterDescription(String clustername) + throws YarnException, IOException { + verifyBindingsDefined(); + validateClusterName(clustername); + return getClusterDescription(clustername); + } + @Override public int actionVersion() { SliderVersionInfo.loadAndPrintVersionInfo(log); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java index 30f6ba9f59..c6cc2d0d54 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java @@ -279,6 +279,17 @@ String actionEcho(String name, ActionEchoArgs args) int actionStatus(String clustername, ActionStatusArgs statusArgs) throws YarnException, IOException; + /** + * Status operation which returns the status object as a string instead of + * printing it to the console or file. + * + * @param clustername cluster name + * @return cluster status details + * @throws YarnException + * @throws IOException + */ + String actionStatus(String clustername) throws YarnException, IOException; + /** * Version Details * @return exit code