big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-5740. Add a new field in Slider status output - lifetime (remaining). Contributed by Jian He

Browse Source
This commit is contained in:
Gour Saha 2016-12-16 12:03:51 -08:00 committed by Jian He
parent 2cea59dc54
commit 2e1fa61253
3 changed files with 103 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
View File

@ -52,6 +52,8 @@
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.services.api.ApplicationApi;
@ -771,7 +773,7 @@ public Response getApplications(@QueryParam("state") String state) {
// Get all applications in a specific state - lighter projection. For full
// detail, call getApplication on a specific app.
Set<String> applications;
Set<ApplicationReport> applications;
try {
if (StringUtils.isNotEmpty(state)) {
ApplicationStatus appStatus = new ApplicationStatus();
@ -793,13 +795,12 @@ public Response getApplications(@QueryParam("state") String state) {
Set<Application> apps = new HashSet<Application>();
if (applications.size() > 0) {
try {
for (String app : applications) {
for (ApplicationReport app : applications) {
Application application = new Application();
// TODO: Need to get lifetime, launch-time and privileged container
// status from YARN
application.setLifetime(null);
application.setLaunchTime(new Date());
application.setName(app);
application.setLifetime(app.getApplicationTimeouts().get(
ApplicationTimeoutType.LIFETIME).getRemainingTime());
application.setLaunchTime(new Date(app.getStartTime()));
application.setName(app.getName());
// Containers not required, setting to null to avoid empty list
application.setContainers(null);
apps.add(application);
@ -930,9 +931,7 @@ private Response populateAppData(Application app, JsonObject appStatus,
app.setLaunchTime(appStatus.get("createTime") == null ? null
: new Date(appStatus.get("createTime").getAsLong()));
// lifetime - set it to unlimited for now
// TODO: Once YARN-3813 and YARN-4205 are available - get it from YARN
app.setLifetime(DEFAULT_UNLIMITED_LIFETIME);
app.setLifetime(queryLifetime(appName));
// Quicklinks
Map<String, String> appQuicklinks = new HashMap<>();
@ -1062,6 +1061,24 @@ private JsonObject jsonGetAsObject(JsonObject object, String key) {
return object.get(key) == null ? null : object.get(key).getAsJsonObject();
}
private long queryLifetime(String appName) {
try {
return invokeSliderClientRunnable(
new SliderClientContextRunnable<Long>() {
@Override
public Long run(SliderClient sliderClient)
throws YarnException, IOException, InterruptedException {
ApplicationReport report = sliderClient.findInstance(appName);
return report.getApplicationTimeouts()
.get(ApplicationTimeoutType.LIFETIME).getRemainingTime();
}
});
} catch (Exception e) {
logger.error("Error when querying lifetime for " + appName, e);
return DEFAULT_UNLIMITED_LIFETIME;
}
}
private JsonObject getSliderApplicationStatus(final String appName)
throws IOException, YarnException, InterruptedException {
@ -1142,36 +1159,37 @@ public Integer run(SliderClient sliderClient) throws YarnException,
});
}
private Set<String> getSliderApplications(final String state)
private Set<ApplicationReport> getSliderApplications(final String state)
throws IOException, YarnException, InterruptedException {
return getSliderApplications(false, state);
}
private Set<String> getSliderApplications(final boolean liveOnly)
private Set<ApplicationReport> getSliderApplications(final boolean liveOnly)
throws IOException, YarnException, InterruptedException {
return getSliderApplications(liveOnly, null);
}
private Set<String> getSliderApplications(final boolean liveOnly,
final String state) throws IOException, YarnException,
InterruptedException {
return invokeSliderClientRunnable(new SliderClientContextRunnable<Set<String>>() {
@Override
public Set<String> run(SliderClient sliderClient) throws YarnException,
IOException, InterruptedException {
Set<String> apps;
ActionListArgs listArgs = new ActionListArgs();
if (liveOnly) {
apps = sliderClient.getApplicationList(null);
} else if (StringUtils.isNotEmpty(state)) {
listArgs.state = state;
apps = sliderClient.getApplicationList(null, listArgs);
} else {
apps = sliderClient.getApplicationList(null, listArgs);
}
return apps;
}
});
private Set<ApplicationReport> getSliderApplications(final boolean liveOnly,
final String state)
throws IOException, YarnException, InterruptedException {
return invokeSliderClientRunnable(
new SliderClientContextRunnable<Set<ApplicationReport>>() {
@Override
public Set<ApplicationReport> run(SliderClient sliderClient)
throws YarnException, IOException, InterruptedException {
Set<ApplicationReport> apps;
ActionListArgs listArgs = new ActionListArgs();
if (liveOnly) {
apps = sliderClient.getApplicationList(null);
} else if (StringUtils.isNotEmpty(state)) {
listArgs.state = state;
apps = sliderClient.getApplicationList(null, listArgs);
} else {
apps = sliderClient.getApplicationList(null, listArgs);
}
return apps;
}
});
}
@DELETE

62
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
View File

@ -55,6 +55,7 @@
import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.LocalResource;
@ -178,6 +179,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.ByteArrayOutputStream;
import java.io.Console;
import java.io.File;
import java.io.FileNotFoundException;
@ -185,6 +187,7 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.InterruptedIOException;
import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.io.StringWriter;
@ -2706,11 +2709,12 @@ public int actionList(String clustername) throws IOException, YarnException {
@Override
public int actionList(String clustername, ActionListArgs args)
throws IOException, YarnException {
Set<String> appInstances = getApplicationList(clustername, args);
// getApplicationList never returns null
return !appInstances.isEmpty() ? EXIT_SUCCESS
: ((appInstances.isEmpty() && isUnset(clustername)) ? EXIT_SUCCESS
: EXIT_FALSE);
Set<ApplicationReport> appInstances = getApplicationList(clustername, args);
if (!appInstances.isEmpty()) {
return EXIT_SUCCESS;
} else {
return EXIT_FALSE;
}
}
/**
@ -2723,8 +2727,8 @@ public int actionList(String clustername, ActionListArgs args)
* @throws IOException
* @throws YarnException
*/
public Set<String> getApplicationList(String clustername) throws IOException,
YarnException {
public Set<ApplicationReport> getApplicationList(String clustername)
throws IOException, YarnException {
ActionListArgs args = new ActionListArgs();
args.live = true;
return getApplicationList(clustername, args);
@ -2743,8 +2747,8 @@ public Set<String> getApplicationList(String clustername) throws IOException,
* @throws UnknownApplicationInstanceException
* if a specific instance was named but it was not found
*/
public Set<String> getApplicationList(String clustername, ActionListArgs args)
throws IOException, YarnException {
public Set<ApplicationReport> getApplicationList(String clustername,
ActionListArgs args) throws IOException, YarnException {
if (args.help) {
actionHelp(ACTION_LIST);
// the above call throws an exception so the return is not really required
@ -2830,13 +2834,13 @@ public Set<String> getApplicationList(String clustername, ActionListArgs args)
}
// at this point there is either the entire list or a stripped down instance
Set<String> listedInstances = new HashSet<String>();
Set<ApplicationReport> listedInstances = new HashSet<ApplicationReport>();
for (String name : persistentInstances.keySet()) {
ApplicationReport report = reportMap.get(name);
if (!listOnlyInState || report != null) {
// list the details if all were requested, or the filtering contained
// a report
listedInstances.add(name);
listedInstances.add(report);
// containers will be non-null when only one instance is requested
String details = instanceDetailsToString(name, report,
containers, version, components, verbose);
@ -3055,7 +3059,7 @@ public YarnAppListClient getYarnAppListClient() {
* @throws YarnException YARN issues
* @throws IOException IO problems
*/
private ApplicationReport findInstance(String appname)
public ApplicationReport findInstance(String appname)
throws YarnException, IOException {
return yarnAppListClient.findInstance(appname);
}
@ -3106,6 +3110,11 @@ private SliderClusterProtocol connect(ApplicationReport app)
@VisibleForTesting
public int actionStatus(String clustername, ActionStatusArgs statusArgs)
throws YarnException, IOException {
if (statusArgs.lifetime) {
queryAndPrintLifetime(clustername);
return EXIT_SUCCESS;
}
ClusterDescription status = verifyAndGetClusterDescription(clustername);
String outfile = statusArgs.getOutput();
if (outfile == null) {
@ -3122,6 +3131,32 @@ public String actionStatus(String clustername)
return verifyAndGetClusterDescription(clustername).toJsonString();
}
private void queryAndPrintLifetime(String appName)
throws YarnException, IOException {
ApplicationReport appReport = findInstance(appName);
if (appReport == null) {
throw new YarnException("No application found for " + appName);
}
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter timeoutStr =
new PrintWriter(new OutputStreamWriter(baos, Charset.forName("UTF-8")));
try {
ApplicationTimeout lifetime = appReport.getApplicationTimeouts()
.get(ApplicationTimeoutType.LIFETIME);
if (lifetime.getRemainingTime() == -1L) {
timeoutStr.append(appName + " has no lifetime configured.");
} else {
timeoutStr.append("\t" + ApplicationTimeoutType.LIFETIME);
timeoutStr.print(" expires at : " + lifetime.getExpiryTime());
timeoutStr.println(
".\tRemaining Time : " + lifetime.getRemainingTime() + " seconds");
}
System.out.println(baos.toString("UTF-8"));
} finally {
timeoutStr.close();
}
}
private ClusterDescription verifyAndGetClusterDescription(String clustername)
throws YarnException, IOException {
verifyBindingsDefined();
@ -3547,7 +3582,8 @@ public String toString() {
* @throws IOException
*/
@VisibleForTesting
public List<ApplicationReport> getApplications() throws YarnException, IOException {
public List<ApplicationReport> getApplications()
throws YarnException, IOException {
return yarnClient.getApplications();
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java
View File

@ -35,6 +35,10 @@ public String getActionName() {
description = "Output file for the status information")
public String output;
@Parameter(names = {ARG_LIFETIME},
description = "Lifetime of the application from the time of request")
public boolean lifetime;
public String getOutput() {
return output;
}