diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 1ca1ba2c9e..e18cd9fb5f 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -353,6 +353,9 @@ Release 2.0.5-beta - UNRELEASED MAPREDUCE-5181. RMCommunicator should not use AMToken from the env. (Vinod Kumar Vavilapalli via sseth) + MAPREDUCE-5178. Update MR App to set progress in ApplicationReport after + YARN-577. (Hitesh Shah via vinodkv) + Release 2.0.4-alpha - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java index c33ab38c15..50bfedd643 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java @@ -89,7 +89,7 @@ private ApplicationReport getUnknownApplicationReport() { // used for a non running job return BuilderUtils.newApplicationReport(unknownAppId, unknownAttemptId, "N/A", "N/A", "N/A", "N/A", 0, null, YarnApplicationState.NEW, "N/A", - "N/A", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A"); + "N/A", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f); } NotRunningJob(ApplicationReport applicationReport, JobState jobState) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java index 8e1be11bfc..4eaddfcf57 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java @@ -413,7 +413,7 @@ private ApplicationReport getFinishedApplicationReport() { return BuilderUtils.newApplicationReport(appId, attemptId, "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null, - "N/A"); + "N/A", 0.0f); } private ApplicationReport getRunningApplicationReport(String host, int port) { @@ -423,7 +423,7 @@ private ApplicationReport getRunningApplicationReport(String host, int port) { return BuilderUtils.newApplicationReport(appId, attemptId, "user", "queue", "appname", host, port, null, YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, FinalApplicationStatus.UNDEFINED, null, - "N/A"); + "N/A", 0.0f); } private ResourceMgrDelegate getRMDelegate() throws YarnRemoteException { diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index dc12a8a57f..a1ce01e7c1 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -181,6 +181,9 @@ Release 2.0.5-beta - UNRELEASED YARN-581. Added a test to verify that app delegation tokens are restored after RM restart. (Jian He via vinodkv) + YARN-577. Add application-progress also to ApplicationReport. (Hitesh Shah + via vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java index db68efde59..26f9978d7a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java @@ -253,4 +253,16 @@ public interface ApplicationReport { @Private @Unstable void setApplicationResourceUsageReport(ApplicationResourceUsageReport appResources); + + /** + * Get the application's progress ( range 0.0 to 1.0 ) + * @return application's progress + */ + @Public + @Stable + float getProgress(); + + @Private + @Unstable + void setProgress(float progress); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java index 69f939c088..57c334a5be 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java @@ -213,6 +213,12 @@ public FinalApplicationStatus getFinalApplicationStatus() { return convertFromProtoFormat(p.getFinalApplicationStatus()); } + @Override + public float getProgress() { + ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; + return p.getProgress(); + } + @Override public void setApplicationId(ApplicationId applicationId) { maybeInitBuilder(); @@ -345,6 +351,12 @@ public void setFinalApplicationStatus(FinalApplicationStatus finishState) { builder.setFinalApplicationStatus(convertToProtoFormat(finishState)); } + @Override + public void setProgress(float progress) { + maybeInitBuilder(); + builder.setProgress(progress); + } + @Override public ApplicationReportProto getProto() { mergeLocalToProto(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index c865c5dda7..a84bf37e39 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -162,6 +162,7 @@ message ApplicationReportProto { optional ApplicationResourceUsageReportProto app_resource_Usage = 16; optional string originalTrackingUrl = 17; optional ApplicationAttemptIdProto currentApplicationAttemptId = 18; + optional float progress = 19; } enum NodeStateProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index d884e769fa..5797e341da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -20,6 +20,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintWriter; +import java.text.DecimalFormat; import java.util.List; import org.apache.commons.cli.CommandLine; @@ -34,7 +35,7 @@ public class ApplicationCLI extends YarnCLI { private static final String APPLICATIONS_PATTERN = - "%30s\t%20s\t%10s\t%10s\t%18s\t%18s\t%35s" + + "%30s\t%20s\t%10s\t%10s\t%18s\t%18s\t%15s\t%35s" + System.getProperty("line.separator"); public static void main(String[] args) throws Exception { @@ -98,12 +99,15 @@ private void listAllApplications() throws YarnRemoteException { writer.println("Total Applications:" + appsReport.size()); writer.printf(APPLICATIONS_PATTERN, "Application-Id", "Application-Name", "User", "Queue", "State", "Final-State", - "Tracking-URL"); + "Progress", "Tracking-URL"); for (ApplicationReport appReport : appsReport) { + DecimalFormat formatter = new DecimalFormat("###.##%"); + String progress = formatter.format(appReport.getProgress()); writer.printf(APPLICATIONS_PATTERN, appReport.getApplicationId(), appReport.getName(), appReport.getUser(), appReport.getQueue(), appReport.getYarnApplicationState(), appReport - .getFinalApplicationStatus(), appReport.getOriginalTrackingUrl()); + .getFinalApplicationStatus(), + progress, appReport.getOriginalTrackingUrl()); } writer.flush(); } @@ -147,6 +151,10 @@ private void printApplicationReport(String applicationId) appReportStr.println(appReport.getStartTime()); appReportStr.print("\tFinish-Time : "); appReportStr.println(appReport.getFinishTime()); + appReportStr.print("\tProgress : "); + DecimalFormat formatter = new DecimalFormat("###.##%"); + String progress = formatter.format(appReport.getProgress()); + appReportStr.println(progress); appReportStr.print("\tState : "); appReportStr.println(appReport.getYarnApplicationState()); appReportStr.print("\tFinal-State : "); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index fbe4809114..716cf2562f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -76,7 +76,7 @@ public void testGetApplicationReport() throws Exception { applicationId, BuilderUtils.newApplicationAttemptId(applicationId, 1), "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, - FinalApplicationStatus.SUCCEEDED, null, "N/A"); + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport); int result = cli.run(new String[] { "-status", applicationId.toString() }); @@ -91,6 +91,7 @@ public void testGetApplicationReport() throws Exception { pw.println("\tQueue : queue"); pw.println("\tStart-Time : 0"); pw.println("\tFinish-Time : 0"); + pw.println("\tProgress : 53.79%"); pw.println("\tState : FINISHED"); pw.println("\tFinal-State : SUCCEEDED"); pw.println("\tTracking-URL : N/A"); @@ -111,7 +112,7 @@ public void testGetAllApplications() throws Exception { applicationId, BuilderUtils.newApplicationAttemptId(applicationId, 1), "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, - FinalApplicationStatus.SUCCEEDED, null, "N/A"); + FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f); List applicationReports = new ArrayList(); applicationReports.add(newApplicationReport); when(client.getApplicationList()).thenReturn(applicationReports); @@ -124,10 +125,12 @@ public void testGetAllApplications() throws Exception { pw.println("Total Applications:1"); pw.print(" Application-Id\t Application-Name"); pw.print("\t User\t Queue\t State\t "); - pw.println("Final-State\t Tracking-URL"); + pw.print("Final-State\t Progress"); + pw.println("\t Tracking-URL"); pw.print(" application_1234_0005\t "); pw.print("appname\t user\t queue\t FINISHED\t "); - pw.println("SUCCEEDED\t N/A"); + pw.print("SUCCEEDED\t 53.79%"); + pw.println("\t N/A"); pw.close(); String appsReportStr = baos.toString("UTF-8"); Assert.assertEquals(appsReportStr, sysOutStream.toString()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java index 8ce9808f27..7dc25de820 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java @@ -333,7 +333,8 @@ public static ApplicationReport newApplicationReport( ClientToken clientToken, YarnApplicationState state, String diagnostics, String url, long startTime, long finishTime, FinalApplicationStatus finalStatus, - ApplicationResourceUsageReport appResources, String origTrackingUrl) { + ApplicationResourceUsageReport appResources, String origTrackingUrl, + float progress) { ApplicationReport report = recordFactory .newRecordInstance(ApplicationReport.class); report.setApplicationId(applicationId); @@ -352,6 +353,7 @@ public static ApplicationReport newApplicationReport( report.setFinalApplicationStatus(finalStatus); report.setApplicationResourceUsageReport(appResources); report.setOriginalTrackingUrl(origTrackingUrl); + report.setProgress(progress); return report; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 7b63cbe804..874232bb16 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -437,6 +437,7 @@ public ApplicationReport createAndGetApplicationReport(boolean allowAccess) { DUMMY_APPLICATION_RESOURCE_USAGE_REPORT; FinalApplicationStatus finishState = getFinalApplicationStatus(); String diags = UNAVAILABLE; + float progress = 0.0f; if (allowAccess) { if (this.currentAttempt != null) { currentApplicationAttemptId = this.currentAttempt.getAppAttemptId(); @@ -446,8 +447,8 @@ public ApplicationReport createAndGetApplicationReport(boolean allowAccess) { host = this.currentAttempt.getHost(); rpcPort = this.currentAttempt.getRpcPort(); appUsageReport = currentAttempt.getApplicationResourceUsageReport(); + progress = currentAttempt.getProgress(); } - diags = this.diagnostics.toString(); } @@ -462,7 +463,7 @@ public ApplicationReport createAndGetApplicationReport(boolean allowAccess) { this.name, host, rpcPort, clientToken, createApplicationState(this.stateMachine.getCurrentState()), diags, trackingUrl, this.startTime, this.finishTime, finishState, - appUsageReport, origTrackingUrl); + appUsageReport, origTrackingUrl, progress); } finally { this.readLock.unlock(); }