From 6779467ab6fcc6a02d0e8c80b138cc9df1aa831e Mon Sep 17 00:00:00 2001 From: Jian He Date: Fri, 17 Apr 2015 15:55:34 -0700 Subject: [PATCH] YARN-3451. Display attempt start time and elapsed time on the web UI. Contributed by Rohith Sharmaks --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../api/records/ApplicationAttemptReport.java | 34 ++++++++++++++++++- .../src/main/proto/yarn_protos.proto | 2 ++ .../yarn/client/ProtocolHATestBase.java | 3 +- .../yarn/client/api/impl/TestYarnClient.java | 3 +- .../hadoop/yarn/client/cli/TestYarnCLI.java | 8 ++--- .../pb/ApplicationAttemptReportPBImpl.java | 24 +++++++++++++ .../server/webapp/dao/AppAttemptInfo.java | 12 +++++++ .../rmapp/attempt/RMAppAttemptImpl.java | 3 +- .../webapp/RMAppAttemptBlock.java | 5 +++ 10 files changed, 89 insertions(+), 8 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 3fbaf66d3f..02cf20f13a 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -142,6 +142,9 @@ Release 2.8.0 - UNRELEASED YARN-2696. Queue sorting in CapacityScheduler should consider node label. (Wangda Tan via jianhe) + YARN-3451. Display attempt start time and elapsed time on the web UI. + (Rohith Sharmaks via jianhe) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptReport.java index b7f9c1b3a0..579705842e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptReport.java @@ -47,7 +47,8 @@ public abstract class ApplicationAttemptReport { public static ApplicationAttemptReport newInstance( ApplicationAttemptId applicationAttemptId, String host, int rpcPort, String url, String oUrl, String diagnostics, - YarnApplicationAttemptState state, ContainerId amContainerId) { + YarnApplicationAttemptState state, ContainerId amContainerId, + long startTime, long finishTime) { ApplicationAttemptReport report = Records.newRecord(ApplicationAttemptReport.class); report.setApplicationAttemptId(applicationAttemptId); @@ -58,9 +59,19 @@ public static ApplicationAttemptReport newInstance( report.setDiagnostics(diagnostics); report.setYarnApplicationAttemptState(state); report.setAMContainerId(amContainerId); + report.setStartTime(startTime); + report.setFinishTime(finishTime); return report; } + public static ApplicationAttemptReport newInstance( + ApplicationAttemptId applicationAttemptId, String host, int rpcPort, + String url, String oUrl, String diagnostics, + YarnApplicationAttemptState state, ContainerId amContainerId) { + return newInstance(applicationAttemptId, host, rpcPort, url, oUrl, + diagnostics, state, amContainerId, 0L, 0L); + } + /** * Get the YarnApplicationAttemptState of the application attempt. * @@ -171,4 +182,25 @@ public abstract void setApplicationAttemptId( @Private @Unstable public abstract void setAMContainerId(ContainerId amContainerId); + + @Public + @Unstable + public abstract long getStartTime(); + + @Private + @Unstable + public abstract void setStartTime(long startTime); + + /** + * Get the finish time of the application. + * + * @return finish time of the application + */ + @Public + @Unstable + public abstract long getFinishTime(); + + @Private + @Unstable + public abstract void setFinishTime(long finishTime); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index a0491fe168..ddd3764756 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -215,6 +215,8 @@ message ApplicationAttemptReportProto { optional YarnApplicationAttemptStateProto yarn_application_attempt_state = 6; optional ContainerIdProto am_container_id = 7; optional string original_tracking_url = 8; + optional int64 startTime = 9; + optional int64 finishTime = 10; } enum NodeStateProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java index 509b470524..903dd94367 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java @@ -700,7 +700,8 @@ public List createFakeQueueUserACLInfoList() { public ApplicationAttemptReport createFakeApplicationAttemptReport() { return ApplicationAttemptReport.newInstance( createFakeApplicationAttemptId(), "localhost", 0, "", "", "", - YarnApplicationAttemptState.RUNNING, createFakeContainerId()); + YarnApplicationAttemptState.RUNNING, createFakeContainerId(), 1000l, + 1200l); } public List diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java index de669f2098..10b9bbbbae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java @@ -593,7 +593,8 @@ private List createAppReports() { "diagnostics", YarnApplicationAttemptState.FINISHED, ContainerId.newContainerId( - newApplicationReport.getCurrentApplicationAttemptId(), 1)); + newApplicationReport.getCurrentApplicationAttemptId(), 1), 0, + 0); appAttempts.add(attempt); ApplicationAttemptReport attempt1 = ApplicationAttemptReport.newInstance( ApplicationAttemptId.newInstance(applicationId, 2), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 003f086f9c..1013958bdf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -145,10 +145,10 @@ public void testGetApplicationAttemptReport() throws Exception { ApplicationId applicationId = ApplicationId.newInstance(1234, 5); ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance( applicationId, 1); - ApplicationAttemptReport attemptReport = ApplicationAttemptReport - .newInstance(attemptId, "host", 124, "url", "oUrl", "diagnostics", - YarnApplicationAttemptState.FINISHED, ContainerId.newContainerId( - attemptId, 1)); + ApplicationAttemptReport attemptReport = + ApplicationAttemptReport.newInstance(attemptId, "host", 124, "url", + "oUrl", "diagnostics", YarnApplicationAttemptState.FINISHED, + ContainerId.newContainerId(attemptId, 1), 1000l, 2000l); when( client .getApplicationAttemptReport(any(ApplicationAttemptId.class))) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptReportPBImpl.java index c3c0221ce7..8a97318f4b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptReportPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptReportPBImpl.java @@ -286,4 +286,28 @@ public void setAMContainerId(ContainerId amContainerId) { builder.clearAmContainerId(); this.amContainerId = amContainerId; } + + @Override + public void setStartTime(long startTime) { + maybeInitBuilder(); + builder.setStartTime(startTime); + } + + @Override + public void setFinishTime(long finishTime) { + maybeInitBuilder(); + builder.setFinishTime(finishTime); + } + + @Override + public long getStartTime() { + ApplicationAttemptReportProtoOrBuilder p = viaProto ? proto : builder; + return p.getStartTime(); + } + + @Override + public long getFinishTime() { + ApplicationAttemptReportProtoOrBuilder p = viaProto ? proto : builder; + return p.getFinishTime(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppAttemptInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppAttemptInfo.java index 1ff70289ab..92ebeb0402 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppAttemptInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppAttemptInfo.java @@ -37,6 +37,8 @@ public class AppAttemptInfo { protected String diagnosticsInfo; protected YarnApplicationAttemptState appAttemptState; protected String amContainerId; + protected long startedTime; + protected long finishedTime; public AppAttemptInfo() { // JAXB needs this @@ -53,6 +55,8 @@ public AppAttemptInfo(ApplicationAttemptReport appAttempt) { if (appAttempt.getAMContainerId() != null) { amContainerId = appAttempt.getAMContainerId().toString(); } + startedTime = appAttempt.getStartTime(); + finishedTime = appAttempt.getFinishTime(); } public String getAppAttemptId() { @@ -87,4 +91,12 @@ public String getAmContainerId() { return amContainerId; } + public long getStartedTime() { + return startedTime; + } + + public long getFinishedTime() { + return finishedTime; + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 1071831263..6d3a1cf6cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -1906,7 +1906,8 @@ public ApplicationAttemptReport createApplicationAttemptReport() { attemptReport = ApplicationAttemptReport.newInstance(this .getAppAttemptId(), this.getHost(), this.getRpcPort(), this .getTrackingUrl(), this.getOriginalTrackingUrl(), this.getDiagnostics(), - YarnApplicationAttemptState .valueOf(this.getState().toString()), amId); + YarnApplicationAttemptState.valueOf(this.getState().toString()), + amId, this.startTime, this.finishTime); } finally { this.readLock.unlock(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java index 506e31f28d..6543e92492 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java @@ -41,6 +41,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; import org.apache.hadoop.yarn.server.webapp.AppAttemptBlock; import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo; +import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV; @@ -205,6 +206,10 @@ protected void generateOverview(ApplicationAttemptReport appAttemptReport, "Application Attempt State:", appAttempt.getAppAttemptState() == null ? UNAVAILABLE : appAttempt .getAppAttemptState()) + ._("Started:", Times.format(appAttempt.getStartedTime())) + ._("Elapsed:", + org.apache.hadoop.util.StringUtils.formatTime(Times.elapsed( + appAttempt.getStartedTime(), appAttempt.getFinishedTime()))) ._( "AM Container:", appAttempt.getAmContainerId() == null || containers == null