From 2e1fa61253787bc0ccb366766fdee605618fe26e Mon Sep 17 00:00:00 2001 From: Gour Saha Date: Fri, 16 Dec 2016 12:03:51 -0800 Subject: [PATCH] YARN-5740. Add a new field in Slider status output - lifetime (remaining). Contributed by Jian He --- .../api/impl/ApplicationApiService.java | 82 +++++++++++-------- .../apache/slider/client/SliderClient.java | 62 +++++++++++--- .../common/params/ActionStatusArgs.java | 4 + 3 files changed, 103 insertions(+), 45 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java index c4f5d4396c..b11da2c17d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java @@ -52,6 +52,8 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.services.api.ApplicationApi; @@ -771,7 +773,7 @@ public Response getApplications(@QueryParam("state") String state) { // Get all applications in a specific state - lighter projection. For full // detail, call getApplication on a specific app. - Set applications; + Set applications; try { if (StringUtils.isNotEmpty(state)) { ApplicationStatus appStatus = new ApplicationStatus(); @@ -793,13 +795,12 @@ public Response getApplications(@QueryParam("state") String state) { Set apps = new HashSet(); if (applications.size() > 0) { try { - for (String app : applications) { + for (ApplicationReport app : applications) { Application application = new Application(); - // TODO: Need to get lifetime, launch-time and privileged container - // status from YARN - application.setLifetime(null); - application.setLaunchTime(new Date()); - application.setName(app); + application.setLifetime(app.getApplicationTimeouts().get( + ApplicationTimeoutType.LIFETIME).getRemainingTime()); + application.setLaunchTime(new Date(app.getStartTime())); + application.setName(app.getName()); // Containers not required, setting to null to avoid empty list application.setContainers(null); apps.add(application); @@ -930,9 +931,7 @@ private Response populateAppData(Application app, JsonObject appStatus, app.setLaunchTime(appStatus.get("createTime") == null ? null : new Date(appStatus.get("createTime").getAsLong())); - // lifetime - set it to unlimited for now - // TODO: Once YARN-3813 and YARN-4205 are available - get it from YARN - app.setLifetime(DEFAULT_UNLIMITED_LIFETIME); + app.setLifetime(queryLifetime(appName)); // Quicklinks Map appQuicklinks = new HashMap<>(); @@ -1062,6 +1061,24 @@ private JsonObject jsonGetAsObject(JsonObject object, String key) { return object.get(key) == null ? null : object.get(key).getAsJsonObject(); } + private long queryLifetime(String appName) { + try { + return invokeSliderClientRunnable( + new SliderClientContextRunnable() { + @Override + public Long run(SliderClient sliderClient) + throws YarnException, IOException, InterruptedException { + ApplicationReport report = sliderClient.findInstance(appName); + return report.getApplicationTimeouts() + .get(ApplicationTimeoutType.LIFETIME).getRemainingTime(); + } + }); + } catch (Exception e) { + logger.error("Error when querying lifetime for " + appName, e); + return DEFAULT_UNLIMITED_LIFETIME; + } + } + private JsonObject getSliderApplicationStatus(final String appName) throws IOException, YarnException, InterruptedException { @@ -1142,36 +1159,37 @@ public Integer run(SliderClient sliderClient) throws YarnException, }); } - private Set getSliderApplications(final String state) + private Set getSliderApplications(final String state) throws IOException, YarnException, InterruptedException { return getSliderApplications(false, state); } - private Set getSliderApplications(final boolean liveOnly) + private Set getSliderApplications(final boolean liveOnly) throws IOException, YarnException, InterruptedException { return getSliderApplications(liveOnly, null); } - private Set getSliderApplications(final boolean liveOnly, - final String state) throws IOException, YarnException, - InterruptedException { - return invokeSliderClientRunnable(new SliderClientContextRunnable>() { - @Override - public Set run(SliderClient sliderClient) throws YarnException, - IOException, InterruptedException { - Set apps; - ActionListArgs listArgs = new ActionListArgs(); - if (liveOnly) { - apps = sliderClient.getApplicationList(null); - } else if (StringUtils.isNotEmpty(state)) { - listArgs.state = state; - apps = sliderClient.getApplicationList(null, listArgs); - } else { - apps = sliderClient.getApplicationList(null, listArgs); - } - return apps; - } - }); + private Set getSliderApplications(final boolean liveOnly, + final String state) + throws IOException, YarnException, InterruptedException { + return invokeSliderClientRunnable( + new SliderClientContextRunnable>() { + @Override + public Set run(SliderClient sliderClient) + throws YarnException, IOException, InterruptedException { + Set apps; + ActionListArgs listArgs = new ActionListArgs(); + if (liveOnly) { + apps = sliderClient.getApplicationList(null); + } else if (StringUtils.isNotEmpty(state)) { + listArgs.state = state; + apps = sliderClient.getApplicationList(null, listArgs); + } else { + apps = sliderClient.getApplicationList(null, listArgs); + } + return apps; + } + }); } @DELETE diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java index ef45d1019b..1c126ac08f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -55,6 +55,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationTimeout; import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LocalResource; @@ -178,6 +179,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.ByteArrayOutputStream; import java.io.Console; import java.io.File; import java.io.FileNotFoundException; @@ -185,6 +187,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.InterruptedIOException; +import java.io.OutputStreamWriter; import java.io.PrintStream; import java.io.PrintWriter; import java.io.StringWriter; @@ -2706,11 +2709,12 @@ public int actionList(String clustername) throws IOException, YarnException { @Override public int actionList(String clustername, ActionListArgs args) throws IOException, YarnException { - Set appInstances = getApplicationList(clustername, args); - // getApplicationList never returns null - return !appInstances.isEmpty() ? EXIT_SUCCESS - : ((appInstances.isEmpty() && isUnset(clustername)) ? EXIT_SUCCESS - : EXIT_FALSE); + Set appInstances = getApplicationList(clustername, args); + if (!appInstances.isEmpty()) { + return EXIT_SUCCESS; + } else { + return EXIT_FALSE; + } } /** @@ -2723,8 +2727,8 @@ public int actionList(String clustername, ActionListArgs args) * @throws IOException * @throws YarnException */ - public Set getApplicationList(String clustername) throws IOException, - YarnException { + public Set getApplicationList(String clustername) + throws IOException, YarnException { ActionListArgs args = new ActionListArgs(); args.live = true; return getApplicationList(clustername, args); @@ -2743,8 +2747,8 @@ public Set getApplicationList(String clustername) throws IOException, * @throws UnknownApplicationInstanceException * if a specific instance was named but it was not found */ - public Set getApplicationList(String clustername, ActionListArgs args) - throws IOException, YarnException { + public Set getApplicationList(String clustername, + ActionListArgs args) throws IOException, YarnException { if (args.help) { actionHelp(ACTION_LIST); // the above call throws an exception so the return is not really required @@ -2830,13 +2834,13 @@ public Set getApplicationList(String clustername, ActionListArgs args) } // at this point there is either the entire list or a stripped down instance - Set listedInstances = new HashSet(); + Set listedInstances = new HashSet(); for (String name : persistentInstances.keySet()) { ApplicationReport report = reportMap.get(name); if (!listOnlyInState || report != null) { // list the details if all were requested, or the filtering contained // a report - listedInstances.add(name); + listedInstances.add(report); // containers will be non-null when only one instance is requested String details = instanceDetailsToString(name, report, containers, version, components, verbose); @@ -3055,7 +3059,7 @@ public YarnAppListClient getYarnAppListClient() { * @throws YarnException YARN issues * @throws IOException IO problems */ - private ApplicationReport findInstance(String appname) + public ApplicationReport findInstance(String appname) throws YarnException, IOException { return yarnAppListClient.findInstance(appname); } @@ -3106,6 +3110,11 @@ private SliderClusterProtocol connect(ApplicationReport app) @VisibleForTesting public int actionStatus(String clustername, ActionStatusArgs statusArgs) throws YarnException, IOException { + if (statusArgs.lifetime) { + queryAndPrintLifetime(clustername); + return EXIT_SUCCESS; + } + ClusterDescription status = verifyAndGetClusterDescription(clustername); String outfile = statusArgs.getOutput(); if (outfile == null) { @@ -3122,6 +3131,32 @@ public String actionStatus(String clustername) return verifyAndGetClusterDescription(clustername).toJsonString(); } + private void queryAndPrintLifetime(String appName) + throws YarnException, IOException { + ApplicationReport appReport = findInstance(appName); + if (appReport == null) { + throw new YarnException("No application found for " + appName); + } + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter timeoutStr = + new PrintWriter(new OutputStreamWriter(baos, Charset.forName("UTF-8"))); + try { + ApplicationTimeout lifetime = appReport.getApplicationTimeouts() + .get(ApplicationTimeoutType.LIFETIME); + if (lifetime.getRemainingTime() == -1L) { + timeoutStr.append(appName + " has no lifetime configured."); + } else { + timeoutStr.append("\t" + ApplicationTimeoutType.LIFETIME); + timeoutStr.print(" expires at : " + lifetime.getExpiryTime()); + timeoutStr.println( + ".\tRemaining Time : " + lifetime.getRemainingTime() + " seconds"); + } + System.out.println(baos.toString("UTF-8")); + } finally { + timeoutStr.close(); + } + } + private ClusterDescription verifyAndGetClusterDescription(String clustername) throws YarnException, IOException { verifyBindingsDefined(); @@ -3547,7 +3582,8 @@ public String toString() { * @throws IOException */ @VisibleForTesting - public List getApplications() throws YarnException, IOException { + public List getApplications() + throws YarnException, IOException { return yarnClient.getApplications(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java index 00178df8f6..6fbd96d3fa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java @@ -35,6 +35,10 @@ public String getActionName() { description = "Output file for the status information") public String output; + @Parameter(names = {ARG_LIFETIME}, + description = "Lifetime of the application from the time of request") + public boolean lifetime; + public String getOutput() { return output; }