YARN-577. Add application-progress also to ApplicationReport. Contributed by Hitesh Shah.

MAPREDUCE-5178. Update MR App to set progress in ApplicationReport after YARN-577. Contributed by Hitesh Shah.


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1475636 13f79535-47bb-0310-9956-ffa450edef68

hadoop-mapreduce-project/CHANGES.txt

@@ -353,6 +353,9 @@ Release 2.0.5-beta - UNRELEASED
     MAPREDUCE-5181. RMCommunicator should not use AMToken from the env.
     (Vinod Kumar Vavilapalli via sseth)
 
+    MAPREDUCE-5178. Update MR App to set progress in ApplicationReport after
+    YARN-577. (Hitesh Shah via vinodkv)
+
 Release 2.0.4-alpha - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java

@@ -89,7 +89,7 @@ private ApplicationReport getUnknownApplicationReport() {
     // used for a non running job
     return BuilderUtils.newApplicationReport(unknownAppId, unknownAttemptId,
         "N/A", "N/A", "N/A", "N/A", 0, null, YarnApplicationState.NEW, "N/A",
-        "N/A", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A");
+        "N/A", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f);
   }
 
   NotRunningJob(ApplicationReport applicationReport, JobState jobState) {

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java

@@ -413,7 +413,7 @@ private ApplicationReport getFinishedApplicationReport() {
     return BuilderUtils.newApplicationReport(appId, attemptId, "user", "queue",
         "appname", "host", 124, null, YarnApplicationState.FINISHED,
         "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null,
-        "N/A");
+        "N/A", 0.0f);
   }
 
   private ApplicationReport getRunningApplicationReport(String host, int port) {
@@ -423,7 +423,7 @@ private ApplicationReport getRunningApplicationReport(String host, int port) {
     return BuilderUtils.newApplicationReport(appId, attemptId, "user", "queue",
         "appname", host, port, null, YarnApplicationState.RUNNING,
         "diagnostics", "url", 0, 0, FinalApplicationStatus.UNDEFINED, null,
-        "N/A");
+        "N/A", 0.0f);
   }
 
   private ResourceMgrDelegate getRMDelegate() throws YarnRemoteException {

hadoop-yarn-project/CHANGES.txt

@@ -181,6 +181,9 @@ Release 2.0.5-beta - UNRELEASED
     YARN-581. Added a test to verify that app delegation tokens are restored
     after RM restart. (Jian He via vinodkv)
 
+    YARN-577. Add application-progress also to ApplicationReport. (Hitesh Shah
+    via vinodkv)
+
   OPTIMIZATIONS
 
   BUG FIXES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java

@@ -253,4 +253,16 @@ public interface ApplicationReport {
   @Private
   @Unstable
   void setApplicationResourceUsageReport(ApplicationResourceUsageReport appResources);
+
+  /**
+   * Get the application's progress ( range 0.0 to 1.0 )
+   * @return application's progress
+   */
+  @Public
+  @Stable
+  float getProgress();
+
+  @Private
+  @Unstable
+  void setProgress(float progress);
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java

@@ -213,6 +213,12 @@ public FinalApplicationStatus getFinalApplicationStatus() {
     return convertFromProtoFormat(p.getFinalApplicationStatus());
   }
 
+  @Override
+  public float getProgress() {
+    ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
+    return p.getProgress();
+  }
+
   @Override
   public void setApplicationId(ApplicationId applicationId) {
     maybeInitBuilder();
@@ -345,6 +351,12 @@ public void setFinalApplicationStatus(FinalApplicationStatus finishState) {
     builder.setFinalApplicationStatus(convertToProtoFormat(finishState));
   }
 
+  @Override
+  public void setProgress(float progress) {
+    maybeInitBuilder();
+    builder.setProgress(progress);
+  }
+
   @Override
   public ApplicationReportProto getProto() {
     mergeLocalToProto();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto

@@ -162,6 +162,7 @@ message ApplicationReportProto {
   optional ApplicationResourceUsageReportProto app_resource_Usage = 16;
   optional string originalTrackingUrl = 17;
   optional ApplicationAttemptIdProto currentApplicationAttemptId = 18;
+  optional float progress = 19;
 }
 
 enum NodeStateProto {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java

@@ -20,6 +20,7 @@
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.PrintWriter;
+import java.text.DecimalFormat;
 import java.util.List;
 
 import org.apache.commons.cli.CommandLine;
@@ -34,7 +35,7 @@
 
 public class ApplicationCLI extends YarnCLI {
   private static final String APPLICATIONS_PATTERN =
-    "%30s\t%20s\t%10s\t%10s\t%18s\t%18s\t%35s" +
+    "%30s\t%20s\t%10s\t%10s\t%18s\t%18s\t%15s\t%35s" +
     System.getProperty("line.separator");
 
   public static void main(String[] args) throws Exception {
@@ -98,12 +99,15 @@ private void listAllApplications() throws YarnRemoteException {
     writer.println("Total Applications:" + appsReport.size());
     writer.printf(APPLICATIONS_PATTERN, "Application-Id",
         "Application-Name", "User", "Queue", "State", "Final-State",
-        "Tracking-URL");
+        "Progress", "Tracking-URL");
     for (ApplicationReport appReport : appsReport) {
+      DecimalFormat formatter = new DecimalFormat("###.##%");
+      String progress = formatter.format(appReport.getProgress());
       writer.printf(APPLICATIONS_PATTERN, appReport.getApplicationId(),
           appReport.getName(), appReport.getUser(), appReport.getQueue(),
           appReport.getYarnApplicationState(), appReport
-              .getFinalApplicationStatus(), appReport.getOriginalTrackingUrl());
+              .getFinalApplicationStatus(),
+          progress, appReport.getOriginalTrackingUrl());
     }
     writer.flush();
   }
@@ -147,6 +151,10 @@ private void printApplicationReport(String applicationId)
       appReportStr.println(appReport.getStartTime());
       appReportStr.print("\tFinish-Time : ");
       appReportStr.println(appReport.getFinishTime());
+      appReportStr.print("\tProgress : ");
+      DecimalFormat formatter = new DecimalFormat("###.##%");
+      String progress = formatter.format(appReport.getProgress());
+      appReportStr.println(progress);
       appReportStr.print("\tState : ");
       appReportStr.println(appReport.getYarnApplicationState());
       appReportStr.print("\tFinal-State : ");

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java

@@ -76,7 +76,7 @@ public void testGetApplicationReport() throws Exception {
         applicationId, BuilderUtils.newApplicationAttemptId(applicationId, 1),
         "user", "queue", "appname", "host", 124, null,
         YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
-        FinalApplicationStatus.SUCCEEDED, null, "N/A");
+        FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f);
     when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
         newApplicationReport);
     int result = cli.run(new String[] { "-status", applicationId.toString() });
@@ -91,6 +91,7 @@ public void testGetApplicationReport() throws Exception {
     pw.println("\tQueue : queue");
     pw.println("\tStart-Time : 0");
     pw.println("\tFinish-Time : 0");
+    pw.println("\tProgress : 53.79%");
     pw.println("\tState : FINISHED");
     pw.println("\tFinal-State : SUCCEEDED");
     pw.println("\tTracking-URL : N/A");
@@ -111,7 +112,7 @@ public void testGetAllApplications() throws Exception {
         applicationId, BuilderUtils.newApplicationAttemptId(applicationId, 1),
         "user", "queue", "appname", "host", 124, null,
         YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
-        FinalApplicationStatus.SUCCEEDED, null, "N/A");
+        FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f);
     List<ApplicationReport> applicationReports = new ArrayList<ApplicationReport>();
     applicationReports.add(newApplicationReport);
     when(client.getApplicationList()).thenReturn(applicationReports);
@@ -124,10 +125,12 @@ public void testGetAllApplications() throws Exception {
     pw.println("Total Applications:1");
     pw.print("                Application-Id\t    Application-Name");
     pw.print("\t      User\t     Queue\t             State\t       ");
-    pw.println("Final-State\t                       Tracking-URL");
+    pw.print("Final-State\t       Progress");
+    pw.println("\t                       Tracking-URL");
     pw.print("         application_1234_0005\t             ");
     pw.print("appname\t      user\t     queue\t          FINISHED\t         ");
-    pw.println("SUCCEEDED\t                                N/A");
+    pw.print("SUCCEEDED\t         53.79%");
+    pw.println("\t                                N/A");
     pw.close();
     String appsReportStr = baos.toString("UTF-8");
     Assert.assertEquals(appsReportStr, sysOutStream.toString());

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java

@@ -333,7 +333,8 @@ public static ApplicationReport newApplicationReport(
       ClientToken clientToken, YarnApplicationState state, String diagnostics,
       String url, long startTime, long finishTime,
       FinalApplicationStatus finalStatus,
-      ApplicationResourceUsageReport appResources, String origTrackingUrl) {
+      ApplicationResourceUsageReport appResources, String origTrackingUrl,
+      float progress) {
     ApplicationReport report = recordFactory
         .newRecordInstance(ApplicationReport.class);
     report.setApplicationId(applicationId);
@@ -352,6 +353,7 @@ public static ApplicationReport newApplicationReport(
     report.setFinalApplicationStatus(finalStatus);
     report.setApplicationResourceUsageReport(appResources);
     report.setOriginalTrackingUrl(origTrackingUrl);
+    report.setProgress(progress);
     return report;
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java

@@ -437,6 +437,7 @@ public ApplicationReport createAndGetApplicationReport(boolean allowAccess) {
           DUMMY_APPLICATION_RESOURCE_USAGE_REPORT;
       FinalApplicationStatus finishState = getFinalApplicationStatus();
       String diags = UNAVAILABLE;
+      float progress = 0.0f;
       if (allowAccess) {
         if (this.currentAttempt != null) {
           currentApplicationAttemptId = this.currentAttempt.getAppAttemptId();
@@ -446,8 +447,8 @@ public ApplicationReport createAndGetApplicationReport(boolean allowAccess) {
           host = this.currentAttempt.getHost();
           rpcPort = this.currentAttempt.getRpcPort();
           appUsageReport = currentAttempt.getApplicationResourceUsageReport();
+          progress = currentAttempt.getProgress();
         }
-
         diags = this.diagnostics.toString();
       }
 
@@ -462,7 +463,7 @@ public ApplicationReport createAndGetApplicationReport(boolean allowAccess) {
           this.name, host, rpcPort, clientToken,
           createApplicationState(this.stateMachine.getCurrentState()), diags,
           trackingUrl, this.startTime, this.finishTime, finishState,
-          appUsageReport, origTrackingUrl);
+          appUsageReport, origTrackingUrl, progress);
     } finally {
       this.readLock.unlock();
     }