From 4ec1cbe86dd9e72bcd2479046ea020d7dfa22794 Mon Sep 17 00:00:00 2001 From: Gour Saha <gourksaha@apache.org> Date: Fri, 14 Oct 2016 17:40:51 -0700 Subject: [PATCH] YARN-5735. Make the service REST API use the app timeout feature YARN-4205. Contributed by Jian He --- .../api/impl/ApplicationApiService.java | 10 ++++-- .../apache/slider/client/SliderClient.java | 33 ++++++++++--------- .../AbstractClusterBuildingActionArgs.java | 5 +++ .../slider/common/params/ActionThawArgs.java | 6 ++++ .../slider/common/params/Arguments.java | 1 + 5 files changed, 36 insertions(+), 19 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java index 21cf1136ec..73df4a11a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java @@ -347,7 +347,7 @@ public class ApplicationApiService implements ApplicationApi { if (queueName != null && queueName.trim().length() > 0) { createArgs.queue = queueName.trim(); } - + createArgs.lifetime = application.getLifetime(); return invokeSliderClientRunnable(new SliderClientContextRunnable<String>() { @Override public String run(SliderClient sliderClient) throws YarnException, @@ -1246,13 +1246,17 @@ public class ApplicationApiService implements ApplicationApi { }); } - private Response startSliderApplication(final String appName) + private Response startSliderApplication(final String appName, Application app) throws IOException, YarnException, InterruptedException { return invokeSliderClientRunnable(new SliderClientContextRunnable<Response>() { @Override public Response run(SliderClient sliderClient) throws YarnException, IOException, InterruptedException { ActionThawArgs thawArgs = new ActionThawArgs(); + if (app.getLifetime() == null) { + app.setLifetime(DEFAULT_UNLIMITED_LIFETIME); + } + thawArgs.lifetime = app.getLifetime(); int returnCode = sliderClient.actionThaw(appName, thawArgs); if (returnCode == 0) { logger.info("Successfully started application {}", appName); @@ -1344,7 +1348,7 @@ public class ApplicationApiService implements ApplicationApi { try { int livenessCheck = getSliderList(appName); if (livenessCheck != 0) { - return startSliderApplication(appName); + return startSliderApplication(appName, updateAppData); } else { logger.info("Application {} is already running", appName); ApplicationStatus applicationStatus = new ApplicationStatus(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java index fe4f1d299c..2840c4b6df 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -54,6 +54,7 @@ import org.apache.hadoop.util.Shell; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.NodeReport; @@ -734,7 +735,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe sliderFileSystem.getFileSystem().delete(clusterDirectory, true); throw e; } - return startCluster(clustername, createArgs); + return startCluster(clustername, createArgs, createArgs.lifetime); } @Override @@ -1960,14 +1961,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe * * @param clustername name of the cluster. * @param launchArgs launch arguments + * @param lifetime * @return the exit code * @throws YarnException * @throws IOException */ - protected int startCluster(String clustername, - LaunchArgsAccessor launchArgs) throws - YarnException, - IOException { + protected int startCluster(String clustername, LaunchArgsAccessor launchArgs, + long lifetime) throws YarnException, IOException { Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername); AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved( clustername, @@ -1975,7 +1975,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe LaunchedApplication launchedApplication = launchApplication(clustername, clusterDirectory, instanceDefinition, - serviceArgs.isDebug()); + serviceArgs.isDebug(), lifetime); if (launchArgs.getOutputFile() != null) { // output file has been requested. Get the app report and serialize it @@ -2044,9 +2044,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } protected AppMasterLauncher setupAppMasterLauncher(String clustername, - Path clusterDirectory, - AggregateConf instanceDefinition, - boolean debugAM) + Path clusterDirectory, AggregateConf instanceDefinition, boolean debugAM, + long lifetime) throws YarnException, IOException{ deployedClusterName = clustername; validateClusterName(clustername); @@ -2119,7 +2118,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe ApplicationId appId = amLauncher.getApplicationId(); // set the application name; amLauncher.setKeepContainersOverRestarts(true); - + // set lifetime in submission context; + Map<ApplicationTimeoutType, Long> appTimeout = new HashMap<>(); + appTimeout.put(ApplicationTimeoutType.LIFETIME, lifetime); + amLauncher.submissionContext.setApplicationTimeouts(appTimeout); int maxAppAttempts = config.getInt(KEY_AM_RESTART_LIMIT, 0); amLauncher.setMaxAppAttempts(maxAppAttempts); @@ -2383,20 +2385,19 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe * @param clusterDirectory cluster dir * @param instanceDefinition the instance definition * @param debugAM enable debug AM options + * @param lifetime * @return the launched application * @throws YarnException * @throws IOException */ - public LaunchedApplication launchApplication(String clustername, - Path clusterDirectory, - AggregateConf instanceDefinition, - boolean debugAM) + public LaunchedApplication launchApplication(String clustername, Path clusterDirectory, + AggregateConf instanceDefinition, boolean debugAM, long lifetime) throws YarnException, IOException { AppMasterLauncher amLauncher = setupAppMasterLauncher(clustername, clusterDirectory, instanceDefinition, - debugAM); + debugAM, lifetime); applicationId = amLauncher.getApplicationId(); log.info("Submitting application {}", applicationId); @@ -3254,7 +3255,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe verifyNoLiveClusters(clustername, "Start"); //start the cluster - return startCluster(clustername, thaw); + return startCluster(clustername, thaw, thaw.lifetime); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java index 2a5eedc692..3cb75e15d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java @@ -102,6 +102,11 @@ public abstract class AbstractClusterBuildingActionArgs extends description = "Queue to submit the application") public String queue; + @Parameter(names = {ARG_LIFETIME}, + description = "Life time of the application since application started at" + + " running state") + public long lifetime; + @ParametersDelegate public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java index b43a14e5e0..2bd856f7eb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java @@ -18,6 +18,7 @@ package org.apache.slider.common.params; +import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; import com.beust.jcommander.ParametersDelegate; @@ -43,6 +44,11 @@ public class ActionThawArgs extends AbstractActionArgs implements @ParametersDelegate LaunchArgsDelegate launchArgs = new LaunchArgsDelegate(); + @Parameter(names = {ARG_LIFETIME}, + description = "Life time of the application since application started at" + + " running state") + public long lifetime; + @Override public String getRmAddress() { return launchArgs.getRmAddress(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/Arguments.java index aec4e267aa..cbf7e59dbb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/Arguments.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/Arguments.java @@ -103,6 +103,7 @@ public interface Arguments { String ARG_PRINCIPAL = "--principal"; String ARG_PROVIDER = "--provider"; String ARG_QUEUE = "--queue"; + String ARG_LIFETIME = "--lifetime"; String ARG_REPLACE_PKG = "--replacepkg"; String ARG_RESOURCE = "--resource"; String ARG_RESOURCES = "--resources";