big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-5735. Make the service REST API use the app timeout feature YARN-4205. Contributed by Jian He

Browse Source
This commit is contained in:
Gour Saha 2016-10-14 17:40:51 -07:00 committed by Jian He
parent 7da243ebe0
commit 4ec1cbe86d
5 changed files with 36 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
View File

@ -347,7 +347,7 @@ private String createSliderApp(Application application,
if (queueName != null && queueName.trim().length() > 0) {
createArgs.queue = queueName.trim();
}
createArgs.lifetime = application.getLifetime();
return invokeSliderClientRunnable(new SliderClientContextRunnable<String>() {
@Override
public String run(SliderClient sliderClient) throws YarnException,
@ -1246,13 +1246,17 @@ public Response run(SliderClient sliderClient) throws YarnException,
});
}
private Response startSliderApplication(final String appName)
private Response startSliderApplication(final String appName, Application app)
throws IOException, YarnException, InterruptedException {
return invokeSliderClientRunnable(new SliderClientContextRunnable<Response>() {
@Override
public Response run(SliderClient sliderClient) throws YarnException,
IOException, InterruptedException {
ActionThawArgs thawArgs = new ActionThawArgs();
if (app.getLifetime() == null) {
app.setLifetime(DEFAULT_UNLIMITED_LIFETIME);
}
thawArgs.lifetime = app.getLifetime();
int returnCode = sliderClient.actionThaw(appName, thawArgs);
if (returnCode == 0) {
logger.info("Successfully started application {}", appName);
@ -1344,7 +1348,7 @@ public Response updateApplication(@PathParam("app_name") String appName,
try {
int livenessCheck = getSliderList(appName);
if (livenessCheck != 0) {
return startSliderApplication(appName);
return startSliderApplication(appName, updateAppData);
} else {
logger.info("Application {} is already running", appName);
ApplicationStatus applicationStatus = new ApplicationStatus();

33
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
View File

@ -54,6 +54,7 @@
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.NodeReport;
@ -734,7 +735,7 @@ public int actionCreate(String clustername, ActionCreateArgs createArgs) throws
sliderFileSystem.getFileSystem().delete(clusterDirectory, true);
throw e;
}
return startCluster(clustername, createArgs);
return startCluster(clustername, createArgs, createArgs.lifetime);
}
@Override
@ -1960,14 +1961,13 @@ public void verifyBindingsDefined() throws BadCommandArgumentsException {
*
* @param clustername name of the cluster.
* @param launchArgs launch arguments
* @param lifetime
* @return the exit code
* @throws YarnException
* @throws IOException
*/
protected int startCluster(String clustername,
LaunchArgsAccessor launchArgs) throws
YarnException,
IOException {
protected int startCluster(String clustername, LaunchArgsAccessor launchArgs,
long lifetime) throws YarnException, IOException {
Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername);
AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved(
clustername,
@ -1975,7 +1975,7 @@ protected int startCluster(String clustername,
LaunchedApplication launchedApplication =
launchApplication(clustername, clusterDirectory, instanceDefinition,
serviceArgs.isDebug());
serviceArgs.isDebug(), lifetime);
if (launchArgs.getOutputFile() != null) {
// output file has been requested. Get the app report and serialize it
@ -2044,9 +2044,8 @@ public AggregateConf loadInstanceDefinition(String name,
}
protected AppMasterLauncher setupAppMasterLauncher(String clustername,
Path clusterDirectory,
AggregateConf instanceDefinition,
boolean debugAM)
Path clusterDirectory, AggregateConf instanceDefinition, boolean debugAM,
long lifetime)
throws YarnException, IOException{
deployedClusterName = clustername;
validateClusterName(clustername);
@ -2119,7 +2118,10 @@ protected AppMasterLauncher setupAppMasterLauncher(String clustername,
ApplicationId appId = amLauncher.getApplicationId();
// set the application name;
amLauncher.setKeepContainersOverRestarts(true);
// set lifetime in submission context;
Map<ApplicationTimeoutType, Long> appTimeout = new HashMap<>();
appTimeout.put(ApplicationTimeoutType.LIFETIME, lifetime);
amLauncher.submissionContext.setApplicationTimeouts(appTimeout);
int maxAppAttempts = config.getInt(KEY_AM_RESTART_LIMIT, 0);
amLauncher.setMaxAppAttempts(maxAppAttempts);
@ -2383,20 +2385,19 @@ protected AppMasterLauncher setupAppMasterLauncher(String clustername,
* @param clusterDirectory cluster dir
* @param instanceDefinition the instance definition
* @param debugAM enable debug AM options
* @param lifetime
* @return the launched application
* @throws YarnException
* @throws IOException
*/
public LaunchedApplication launchApplication(String clustername,
Path clusterDirectory,
AggregateConf instanceDefinition,
boolean debugAM)
public LaunchedApplication launchApplication(String clustername, Path clusterDirectory,
AggregateConf instanceDefinition, boolean debugAM, long lifetime)
throws YarnException, IOException {
AppMasterLauncher amLauncher = setupAppMasterLauncher(clustername,
clusterDirectory,
instanceDefinition,
debugAM);
debugAM, lifetime);
applicationId = amLauncher.getApplicationId();
log.info("Submitting application {}", applicationId);
@ -3254,7 +3255,7 @@ public int actionThaw(String clustername, ActionThawArgs thaw) throws YarnExcept
verifyNoLiveClusters(clustername, "Start");
//start the cluster
return startCluster(clustername, thaw);
return startCluster(clustername, thaw, thaw.lifetime);
}
/**

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
View File

@ -102,6 +102,11 @@ public abstract class AbstractClusterBuildingActionArgs extends
description = "Queue to submit the application")
public String queue;
@Parameter(names = {ARG_LIFETIME},
description = "Life time of the application since application started at"
+ " running state")
public long lifetime;
@ParametersDelegate
public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate();

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java
View File

@ -18,6 +18,7 @@
package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.beust.jcommander.ParametersDelegate;
@ -43,6 +44,11 @@ public int getWaittime() {
@ParametersDelegate
LaunchArgsDelegate launchArgs = new LaunchArgsDelegate();
@Parameter(names = {ARG_LIFETIME},
description = "Life time of the application since application started at"
+ " running state")
public long lifetime;
@Override
public String getRmAddress() {
return launchArgs.getRmAddress();

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
View File

@ -103,6 +103,7 @@ public interface Arguments {
String ARG_PRINCIPAL = "--principal";
String ARG_PROVIDER = "--provider";
String ARG_QUEUE = "--queue";
String ARG_LIFETIME = "--lifetime";
String ARG_REPLACE_PKG = "--replacepkg";
String ARG_RESOURCE = "--resource";
String ARG_RESOURCES = "--resources";