big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-577. Add application-progress also to ApplicationReport. Contributed by Hitesh Shah.

Browse Source 
MAPREDUCE-5178. Update MR App to set progress in ApplicationReport after YARN-577. Contributed by Hitesh Shah.


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1475636 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2013-04-25 05:42:22 +00:00
parent 16ef8dd3a5
commit 448df18646
11 changed files with 58 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -353,6 +353,9 @@ Release 2.0.5-beta - UNRELEASED
MAPREDUCE-5181. RMCommunicator should not use AMToken from the env.
(Vinod Kumar Vavilapalli via sseth)
MAPREDUCE-5178. Update MR App to set progress in ApplicationReport after
YARN-577. (Hitesh Shah via vinodkv)
Release 2.0.4-alpha - UNRELEASED
INCOMPATIBLE CHANGES

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java
View File

@ -89,7 +89,7 @@ private ApplicationReport getUnknownApplicationReport() {
// used for a non running job
return BuilderUtils.newApplicationReport(unknownAppId, unknownAttemptId,
"N/A", "N/A", "N/A", "N/A", 0, null, YarnApplicationState.NEW, "N/A",
"N/A", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A");
"N/A", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f);
}
NotRunningJob(ApplicationReport applicationReport, JobState jobState) {

4
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
View File

@ -413,7 +413,7 @@ private ApplicationReport getFinishedApplicationReport() {
return BuilderUtils.newApplicationReport(appId, attemptId, "user", "queue",
"appname", "host", 124, null, YarnApplicationState.FINISHED,
"diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null,
"N/A");
"N/A", 0.0f);
}
private ApplicationReport getRunningApplicationReport(String host, int port) {
@ -423,7 +423,7 @@ private ApplicationReport getRunningApplicationReport(String host, int port) {
return BuilderUtils.newApplicationReport(appId, attemptId, "user", "queue",
"appname", host, port, null, YarnApplicationState.RUNNING,
"diagnostics", "url", 0, 0, FinalApplicationStatus.UNDEFINED, null,
"N/A");
"N/A", 0.0f);
}
private ResourceMgrDelegate getRMDelegate() throws YarnRemoteException {

3
hadoop-yarn-project/CHANGES.txt
View File

@ -181,6 +181,9 @@ Release 2.0.5-beta - UNRELEASED
YARN-581. Added a test to verify that app delegation tokens are restored
after RM restart. (Jian He via vinodkv)
YARN-577. Add application-progress also to ApplicationReport. (Hitesh Shah
via vinodkv)
OPTIMIZATIONS
BUG FIXES

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
View File

@ -253,4 +253,16 @@ public interface ApplicationReport {
@Private
@Unstable
void setApplicationResourceUsageReport(ApplicationResourceUsageReport appResources);
/**
* Get the application's progress ( range 0.0 to 1.0 )
* @return application's progress
*/
@Public
@Stable
float getProgress();
@Private
@Unstable
void setProgress(float progress);
}

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
View File

@ -213,6 +213,12 @@ public FinalApplicationStatus getFinalApplicationStatus() {
return convertFromProtoFormat(p.getFinalApplicationStatus());
}
@Override
public float getProgress() {
ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
return p.getProgress();
}
@Override
public void setApplicationId(ApplicationId applicationId) {
maybeInitBuilder();
@ -345,6 +351,12 @@ public void setFinalApplicationStatus(FinalApplicationStatus finishState) {
builder.setFinalApplicationStatus(convertToProtoFormat(finishState));
}
@Override
public void setProgress(float progress) {
maybeInitBuilder();
builder.setProgress(progress);
}
@Override
public ApplicationReportProto getProto() {
mergeLocalToProto();

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
View File

@ -162,6 +162,7 @@ message ApplicationReportProto {
optional ApplicationResourceUsageReportProto app_resource_Usage = 16;
optional string originalTrackingUrl = 17;
optional ApplicationAttemptIdProto currentApplicationAttemptId = 18;
optional float progress = 19;
}
enum NodeStateProto {

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
View File

@ -20,6 +20,7 @@
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.text.DecimalFormat;
import java.util.List;
import org.apache.commons.cli.CommandLine;
@ -34,7 +35,7 @@
public class ApplicationCLI extends YarnCLI {
private static final String APPLICATIONS_PATTERN =
"%30s\t%20s\t%10s\t%10s\t%18s\t%18s\t%35s" +
"%30s\t%20s\t%10s\t%10s\t%18s\t%18s\t%15s\t%35s" +
System.getProperty("line.separator");
public static void main(String[] args) throws Exception {
@ -98,12 +99,15 @@ private void listAllApplications() throws YarnRemoteException {
writer.println("Total Applications:" + appsReport.size());
writer.printf(APPLICATIONS_PATTERN, "Application-Id",
"Application-Name", "User", "Queue", "State", "Final-State",
"Tracking-URL");
"Progress", "Tracking-URL");
for (ApplicationReport appReport : appsReport) {
DecimalFormat formatter = new DecimalFormat("###.##%");
String progress = formatter.format(appReport.getProgress());
writer.printf(APPLICATIONS_PATTERN, appReport.getApplicationId(),
appReport.getName(), appReport.getUser(), appReport.getQueue(),
appReport.getYarnApplicationState(), appReport
.getFinalApplicationStatus(), appReport.getOriginalTrackingUrl());
.getFinalApplicationStatus(),
progress, appReport.getOriginalTrackingUrl());
}
writer.flush();
}
@ -147,6 +151,10 @@ private void printApplicationReport(String applicationId)
appReportStr.println(appReport.getStartTime());
appReportStr.print("\tFinish-Time : ");
appReportStr.println(appReport.getFinishTime());
appReportStr.print("\tProgress : ");
DecimalFormat formatter = new DecimalFormat("###.##%");
String progress = formatter.format(appReport.getProgress());
appReportStr.println(progress);
appReportStr.print("\tState : ");
appReportStr.println(appReport.getYarnApplicationState());
appReportStr.print("\tFinal-State : ");

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
View File

@ -76,7 +76,7 @@ public void testGetApplicationReport() throws Exception {
applicationId, BuilderUtils.newApplicationAttemptId(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
FinalApplicationStatus.SUCCEEDED, null, "N/A");
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f);
when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
newApplicationReport);
int result = cli.run(new String[] { "-status", applicationId.toString() });
@ -91,6 +91,7 @@ public void testGetApplicationReport() throws Exception {
pw.println("\tQueue : queue");
pw.println("\tStart-Time : 0");
pw.println("\tFinish-Time : 0");
pw.println("\tProgress : 53.79%");
pw.println("\tState : FINISHED");
pw.println("\tFinal-State : SUCCEEDED");
pw.println("\tTracking-URL : N/A");
@ -111,7 +112,7 @@ public void testGetAllApplications() throws Exception {
applicationId, BuilderUtils.newApplicationAttemptId(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
FinalApplicationStatus.SUCCEEDED, null, "N/A");
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f);
List<ApplicationReport> applicationReports = new ArrayList<ApplicationReport>();
applicationReports.add(newApplicationReport);
when(client.getApplicationList()).thenReturn(applicationReports);
@ -124,10 +125,12 @@ public void testGetAllApplications() throws Exception {
pw.println("Total Applications:1");
pw.print(" Application-Id\t Application-Name");
pw.print("\t User\t Queue\t State\t ");
pw.println("Final-State\t Tracking-URL");
pw.print("Final-State\t Progress");
pw.println("\t Tracking-URL");
pw.print(" application_1234_0005\t ");
pw.print("appname\t user\t queue\t FINISHED\t ");
pw.println("SUCCEEDED\t N/A");
pw.print("SUCCEEDED\t 53.79%");
pw.println("\t N/A");
pw.close();
String appsReportStr = baos.toString("UTF-8");
Assert.assertEquals(appsReportStr, sysOutStream.toString());

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java
View File

@ -333,7 +333,8 @@ public static ApplicationReport newApplicationReport(
ClientToken clientToken, YarnApplicationState state, String diagnostics,
String url, long startTime, long finishTime,
FinalApplicationStatus finalStatus,
ApplicationResourceUsageReport appResources, String origTrackingUrl) {
ApplicationResourceUsageReport appResources, String origTrackingUrl,
float progress) {
ApplicationReport report = recordFactory
.newRecordInstance(ApplicationReport.class);
report.setApplicationId(applicationId);
@ -352,6 +353,7 @@ public static ApplicationReport newApplicationReport(
report.setFinalApplicationStatus(finalStatus);
report.setApplicationResourceUsageReport(appResources);
report.setOriginalTrackingUrl(origTrackingUrl);
report.setProgress(progress);
return report;
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
View File

@ -437,6 +437,7 @@ public ApplicationReport createAndGetApplicationReport(boolean allowAccess) {
DUMMY_APPLICATION_RESOURCE_USAGE_REPORT;
FinalApplicationStatus finishState = getFinalApplicationStatus();
String diags = UNAVAILABLE;
float progress = 0.0f;
if (allowAccess) {
if (this.currentAttempt != null) {
currentApplicationAttemptId = this.currentAttempt.getAppAttemptId();
@ -446,8 +447,8 @@ public ApplicationReport createAndGetApplicationReport(boolean allowAccess) {
host = this.currentAttempt.getHost();
rpcPort = this.currentAttempt.getRpcPort();
appUsageReport = currentAttempt.getApplicationResourceUsageReport();
progress = currentAttempt.getProgress();
}
diags = this.diagnostics.toString();
}
@ -462,7 +463,7 @@ public ApplicationReport createAndGetApplicationReport(boolean allowAccess) {
this.name, host, rpcPort, clientToken,
createApplicationState(this.stateMachine.getCurrentState()), diags,
trackingUrl, this.startTime, this.finishTime, finishState,
appUsageReport, origTrackingUrl);
appUsageReport, origTrackingUrl, progress);
} finally {
this.readLock.unlock();
}