From bb92bfb4ef96baa234966b60e464d1773fbf3f22 Mon Sep 17 00:00:00 2001 From: Haibo Chen Date: Tue, 17 Apr 2018 11:15:20 -0700 Subject: [PATCH] YARN-7088. Add application launch time to Resource Manager REST API. (Kanwaljeet Sachdev via Haibo Chen) --- .../apache/hadoop/mapred/NotRunningJob.java | 2 +- .../mapred/TestClientServiceDelegate.java | 4 +- .../apache/hadoop/mapred/TestYARNRunner.java | 3 +- .../yarn/api/records/ApplicationReport.java | 45 +++++++++++++++++-- .../src/main/proto/yarn_protos.proto | 1 + .../yarn/client/ProtocolHATestBase.java | 2 +- .../yarn/client/api/impl/TestAHSClient.java | 8 ++-- .../yarn/client/api/impl/TestYarnClient.java | 8 ++-- .../hadoop/yarn/client/cli/TestYarnCLI.java | 42 ++++++++--------- .../impl/pb/ApplicationReportPBImpl.java | 11 +++++ .../hadoop/yarn/api/TestApplicatonReport.java | 2 +- .../ApplicationHistoryManagerImpl.java | 2 +- .../yarn/server/utils/BuilderUtils.java | 3 +- .../hadoop/yarn/server/webapp/AppBlock.java | 7 ++- .../hadoop/yarn/server/webapp/AppsBlock.java | 5 ++- .../yarn/server/webapp/WebPageUtils.java | 2 +- .../yarn/server/webapp/dao/AppInfo.java | 6 +++ .../amlauncher/AMLauncher.java | 2 +- .../records/ApplicationStateData.java | 24 ++++++++-- .../impl/pb/ApplicationStateDataPBImpl.java | 13 ++++++ .../server/resourcemanager/rmapp/RMApp.java | 10 ++++- .../resourcemanager/rmapp/RMAppEvent.java | 6 +++ .../resourcemanager/rmapp/RMAppEventType.java | 1 + .../resourcemanager/rmapp/RMAppImpl.java | 41 +++++++++++++++-- .../rmapp/attempt/RMAppAttemptEvent.java | 7 +++ .../rmapp/attempt/RMAppAttemptImpl.java | 4 +- .../webapp/FairSchedulerAppsBlock.java | 2 + .../resourcemanager/webapp/RMAppsBlock.java | 2 + .../resourcemanager/webapp/dao/AppInfo.java | 6 +++ ...yarn_server_resourcemanager_recovery.proto | 1 + .../applicationsmanager/MockAsm.java | 15 ++++++- .../recovery/RMStateStoreTestBase.java | 4 +- .../recovery/TestFSRMStateStore.java | 2 +- .../recovery/TestZKRMStateStore.java | 2 +- .../resourcemanager/rmapp/MockRMApp.java | 6 +++ .../rmapp/TestRMAppTransitions.java | 2 +- .../resourcemanager/webapp/TestRMWebApp.java | 2 +- .../webapp/TestRMWebServicesApps.java | 14 +++--- 38 files changed, 249 insertions(+), 70 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java index 90635a6885..bfdc6db8ab 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java @@ -89,7 +89,7 @@ private ApplicationReport getUnknownApplicationReport() { // used for a non running job return ApplicationReport.newInstance(unknownAppId, unknownAttemptId, "N/A", "N/A", "N/A", "N/A", 0, null, YarnApplicationState.NEW, "N/A", - "N/A", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f, + "N/A", 0, 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java index b85f18db4e..275456df83 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java @@ -515,7 +515,7 @@ private ApplicationReport getFinishedApplicationReport() { appId, 0); return ApplicationReport.newInstance(appId, attemptId, "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, - "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null, + "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null); } @@ -525,7 +525,7 @@ private ApplicationReport getRunningApplicationReport(String host, int port) { appId, 0); return ApplicationReport.newInstance(appId, attemptId, "user", "queue", "appname", host, port, null, YarnApplicationState.RUNNING, "diagnostics", - "url", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f, + "url", 0, 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java index 932e808bf7..babf22b639 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java @@ -272,7 +272,8 @@ public ClientServiceDelegate answer(InvocationOnMock invocation) .thenReturn( ApplicationReport.newInstance(appId, null, "tmp", "tmp", "tmp", "tmp", 0, null, YarnApplicationState.FINISHED, "tmp", "tmp", - 0l, 0l, FinalApplicationStatus.SUCCEEDED, null, null, 0f, + 0L, 0L, 0L, + FinalApplicationStatus.SUCCEEDED, null, null, 0f, "tmp", null)); yarnRunner.killJob(jobId); verify(clientDelegate).killJob(jobId); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java index 962bbba407..711424b543 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java @@ -58,7 +58,8 @@ public static ApplicationReport newInstance(ApplicationId applicationId, ApplicationAttemptId applicationAttemptId, String user, String queue, String name, String host, int rpcPort, Token clientToAMToken, YarnApplicationState state, String diagnostics, String url, - long startTime, long finishTime, FinalApplicationStatus finalStatus, + long startTime, long launchTime, long finishTime, + FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, float progress, String applicationType, Token amRmToken) { ApplicationReport report = Records.newRecord(ApplicationReport.class); @@ -74,6 +75,7 @@ public static ApplicationReport newInstance(ApplicationId applicationId, report.setDiagnostics(diagnostics); report.setTrackingUrl(url); report.setStartTime(startTime); + report.setLaunchTime(launchTime); report.setFinishTime(finishTime); report.setFinalApplicationStatus(finalStatus); report.setApplicationResourceUsageReport(appResources); @@ -84,13 +86,40 @@ public static ApplicationReport newInstance(ApplicationId applicationId, return report; } + + @Private + @Unstable + public static ApplicationReport newInstance(ApplicationId applicationId, + ApplicationAttemptId applicationAttemptId, String user, String queue, + String name, String host, int rpcPort, Token clientToAMToken, + YarnApplicationState state, String diagnostics, String url, + long startTime, long finishTime, + FinalApplicationStatus finalStatus, + ApplicationResourceUsageReport appResources, String origTrackingUrl, + float progress, String applicationType, Token amRmToken, + Set tags, boolean unmanagedApplication, Priority priority, + String appNodeLabelExpression, String amNodeLabelExpression) { + ApplicationReport report = + newInstance(applicationId, applicationAttemptId, user, queue, name, + host, rpcPort, clientToAMToken, state, diagnostics, url, + startTime, 0, finishTime, finalStatus, appResources, + origTrackingUrl, progress, applicationType, amRmToken); + report.setApplicationTags(tags); + report.setUnmanagedApp(unmanagedApplication); + report.setPriority(priority); + report.setAppNodeLabelExpression(appNodeLabelExpression); + report.setAmNodeLabelExpression(amNodeLabelExpression); + return report; + } + @Private @Unstable public static ApplicationReport newInstance(ApplicationId applicationId, ApplicationAttemptId applicationAttemptId, String user, String queue, String name, String host, int rpcPort, Token clientToAMToken, YarnApplicationState state, String diagnostics, String url, - long startTime, long finishTime, FinalApplicationStatus finalStatus, + long startTime, long launchTime, long finishTime, + FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, float progress, String applicationType, Token amRmToken, Set tags, boolean unmanagedApplication, Priority priority, @@ -98,8 +127,8 @@ public static ApplicationReport newInstance(ApplicationId applicationId, ApplicationReport report = newInstance(applicationId, applicationAttemptId, user, queue, name, host, rpcPort, clientToAMToken, state, diagnostics, url, startTime, - finishTime, finalStatus, appResources, origTrackingUrl, progress, - applicationType, amRmToken); + launchTime, finishTime, finalStatus, appResources, + origTrackingUrl, progress, applicationType, amRmToken); report.setApplicationTags(tags); report.setUnmanagedApp(unmanagedApplication); report.setPriority(priority); @@ -282,6 +311,14 @@ public static ApplicationReport newInstance(ApplicationId applicationId, @Unstable public abstract void setStartTime(long startTime); + @Private + @Unstable + public abstract void setLaunchTime(long setLaunchTime); + + @Public + @Unstable + public abstract long getLaunchTime(); + /** * Get the finish time of the application. * @return finish time of the application diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 5e200dc52d..d6138e865f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -281,6 +281,7 @@ message ApplicationReportProto { optional string appNodeLabelExpression = 24; optional string amNodeLabelExpression = 25; repeated AppTimeoutsMapProto appTimeouts = 26; + optional int64 launchTime = 27; } message AppTimeoutsMapProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java index 7937b1594b..474ae78823 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java @@ -626,7 +626,7 @@ public ApplicationReport createFakeAppReport() { ApplicationReport.newInstance(appId, attemptId, "fakeUser", "fakeQueue", "fakeApplicationName", "localhost", 0, null, YarnApplicationState.FINISHED, "fake an application report", "", - 1000L, 1200L, FinalApplicationStatus.FAILED, null, "", 50f, + 1000L, 1000L, 1200L, FinalApplicationStatus.FAILED, null, "", 50f, "fakeApplicationType", null); return report; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java index c3e3c41477..f0e3ca2a4c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java @@ -332,7 +332,7 @@ private void createAppReports() { ApplicationReport.newInstance(applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); List applicationReports = @@ -389,7 +389,7 @@ private void createAppReports() { ApplicationReport.newInstance(applicationId2, ApplicationAttemptId.newInstance(applicationId2, 2), "user2", "queue2", "appname2", "host2", 125, null, - YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, + YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, 2, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN", null); applicationReports.add(newApplicationReport2); @@ -399,7 +399,7 @@ private void createAppReports() { ApplicationReport.newInstance(applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3), "user3", "queue3", "appname3", "host3", 126, null, - YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, + YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, 3, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", null); applicationReports.add(newApplicationReport3); @@ -409,7 +409,7 @@ private void createAppReports() { ApplicationReport.newInstance(applicationId4, ApplicationAttemptId.newInstance(applicationId4, 4), "user4", "queue4", "appname4", "host4", 127, null, - YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, + YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, 4, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE", null); applicationReports.add(newApplicationReport4); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java index f6e305fab7..b84b49c734 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java @@ -737,7 +737,7 @@ private List createAppReports() { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); List applicationReports = new ArrayList(); applicationReports.add(newApplicationReport); @@ -812,7 +812,7 @@ private List createAppReports() { ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( applicationId2, ApplicationAttemptId.newInstance(applicationId2, 2), "user2", "queue2", "appname2", "host2", 125, null, - YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, + YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, 2, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN", null); applicationReports.add(newApplicationReport2); @@ -821,7 +821,7 @@ private List createAppReports() { ApplicationReport newApplicationReport3 = ApplicationReport.newInstance( applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3), "user3", "queue3", "appname3", "host3", 126, null, - YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, + YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, 3, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", null); applicationReports.add(newApplicationReport3); @@ -832,7 +832,7 @@ private List createAppReports() { applicationId4, ApplicationAttemptId.newInstance(applicationId4, 4), "user4", "queue4", "appname4", "host4", 127, null, - YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, + YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, 4, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE", null); applicationReports.add(newApplicationReport4); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 1f6488db9e..82a20ebec5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -136,7 +136,7 @@ public void testGetApplicationReport() throws Exception { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN", null, null, false, Priority.newInstance(0), "high-mem", "high-mem"); newApplicationReport.setLogAggregationStatus(LogAggregationStatus.SUCCEEDED); @@ -383,7 +383,7 @@ public void testGetApplications() throws Exception { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, Sets.newHashSet("tag1", "tag3"), false, Priority.UNDEFINED, "", ""); List applicationReports = @@ -394,7 +394,7 @@ public void testGetApplications() throws Exception { ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( applicationId2, ApplicationAttemptId.newInstance(applicationId2, 2), "user2", "queue2", "appname2", "host2", 125, null, - YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, + YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, 2, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN", null, Sets.newHashSet("tag2", "tag3"), false, Priority.UNDEFINED, "", ""); @@ -404,7 +404,7 @@ public void testGetApplications() throws Exception { ApplicationReport newApplicationReport3 = ApplicationReport.newInstance( applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3), "user3", "queue3", "appname3", "host3", 126, null, - YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, + YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, 3, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", null, Sets.newHashSet("tag1", "tag4"), false, Priority.UNDEFINED, "", ""); @@ -414,7 +414,7 @@ public void testGetApplications() throws Exception { ApplicationReport newApplicationReport4 = ApplicationReport.newInstance( applicationId4, ApplicationAttemptId.newInstance(applicationId4, 4), "user4", "queue4", "appname4", "host4", 127, null, - YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, + YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, 4, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE", null, Sets.newHashSet("tag1"), false, Priority.UNDEFINED, "", ""); @@ -424,7 +424,7 @@ public void testGetApplications() throws Exception { ApplicationReport newApplicationReport5 = ApplicationReport.newInstance( applicationId5, ApplicationAttemptId.newInstance(applicationId5, 5), "user5", "queue5", "appname5", "host5", 128, null, - YarnApplicationState.ACCEPTED, "diagnostics5", "url5", 5, 5, + YarnApplicationState.ACCEPTED, "diagnostics5", "url5", 5, 5, 5, FinalApplicationStatus.KILLED, null, "N/A", 0.93789f, "HIVE", null, Sets.newHashSet("tag2", "tag4"), false, Priority.UNDEFINED, "", ""); applicationReports.add(newApplicationReport5); @@ -433,7 +433,7 @@ public void testGetApplications() throws Exception { ApplicationReport newApplicationReport6 = ApplicationReport.newInstance( applicationId6, ApplicationAttemptId.newInstance(applicationId6, 6), "user6", "queue6", "appname6", "host6", 129, null, - YarnApplicationState.SUBMITTED, "diagnostics6", "url6", 6, 6, + YarnApplicationState.SUBMITTED, "diagnostics6", "url6", 6, 6, 6, FinalApplicationStatus.KILLED, null, "N/A", 0.99789f, "PIG", null, new HashSet(), false, Priority.UNDEFINED, "", ""); applicationReports.add(newApplicationReport6); @@ -1007,7 +1007,7 @@ public void testKillApplication() throws Exception { ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport2); @@ -1020,7 +1020,7 @@ public void testKillApplication() throws Exception { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport); @@ -1059,12 +1059,12 @@ public void testKillApplications() throws Exception { ApplicationReport newApplicationReport1 = ApplicationReport.newInstance( applicationId1, ApplicationAttemptId.newInstance(applicationId1, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( applicationId2, ApplicationAttemptId.newInstance(applicationId2, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.34344f, "YARN", null); when(client.getApplicationReport(applicationId1)).thenReturn( newApplicationReport1); @@ -1084,12 +1084,12 @@ public void testKillApplications() throws Exception { ApplicationReport newApplicationReport3 = ApplicationReport.newInstance( applicationId1, ApplicationAttemptId.newInstance(applicationId1, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); ApplicationReport newApplicationReport4 = ApplicationReport.newInstance( applicationId2, ApplicationAttemptId.newInstance(applicationId2, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53345f, "YARN", null); when(client.getApplicationReport(applicationId1)).thenReturn( newApplicationReport3); @@ -1127,7 +1127,7 @@ public void testKillApplications() throws Exception { ApplicationReport newApplicationReport5 = ApplicationReport.newInstance( applicationId1, ApplicationAttemptId.newInstance(applicationId1, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53345f, "YARN", null); when(client.getApplicationReport(applicationId1)).thenReturn( newApplicationReport5); @@ -1154,12 +1154,12 @@ public void testKillApplicationsOfDifferentEndStates() throws Exception { ApplicationReport newApplicationReport5 = ApplicationReport.newInstance( applicationId1, ApplicationAttemptId.newInstance(applicationId1, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); ApplicationReport newApplicationReport6 = ApplicationReport.newInstance( applicationId2, ApplicationAttemptId.newInstance(applicationId2, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53345f, "YARN", null); when(client.getApplicationReport(applicationId1)).thenReturn( newApplicationReport5); @@ -1182,7 +1182,7 @@ public void testMoveApplicationAcrossQueues() throws Exception { ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport2); @@ -1197,7 +1197,7 @@ public void testMoveApplicationAcrossQueues() throws Exception { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( newApplicationReport); @@ -1232,7 +1232,7 @@ public void testMoveApplicationAcrossQueuesWithNewCommand() throws Exception { ApplicationReport newApplicationReport2 = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, + YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))) .thenReturn(newApplicationReport2); @@ -1247,7 +1247,7 @@ public void testMoveApplicationAcrossQueuesWithNewCommand() throws Exception { ApplicationReport newApplicationReport = ApplicationReport.newInstance( applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))) .thenReturn(newApplicationReport); @@ -2015,7 +2015,7 @@ public void testUpdateApplicationPriority() throws Exception { ApplicationReport.newInstance(applicationId, ApplicationAttemptId.newInstance(applicationId, 1), "user", "queue", "appname", "host", 124, null, - YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, + YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.53789f, "YARN", null); when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java index 036fa907fe..ddd9accd89 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java @@ -220,6 +220,17 @@ public long getStartTime() { return p.getStartTime(); } + @Override + public long getLaunchTime() { + ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; + return p.getLaunchTime(); + } + + @Override + public void setLaunchTime(long launchTime) { + maybeInitBuilder(); + builder.setLaunchTime(launchTime); + } @Override public long getFinishTime() { ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java index 46fc4d58d9..ea39a4ccdb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java @@ -58,7 +58,7 @@ protected static ApplicationReport createApplicationReport( ApplicationReport appReport = ApplicationReport.newInstance(appId, appAttemptId, "user", "queue", "appname", "host", 124, null, YarnApplicationState.FINISHED, - "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null, + "diagnostics", "url", 0, 0, 0, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, null, false, Priority.newInstance(0),"",""); return appReport; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java index b8931d81ab..3734e39d91 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java @@ -137,7 +137,7 @@ private ApplicationReport convertToApplicationReport( currentApplicationAttemptId, appHistory.getUser(), appHistory.getQueue(), appHistory.getApplicationName(), host, rpcPort, null, appHistory.getYarnApplicationState(), appHistory.getDiagnosticsInfo(), - trackingUrl, appHistory.getStartTime(), appHistory.getFinishTime(), + trackingUrl, appHistory.getStartTime(), 0, appHistory.getFinishTime(), appHistory.getFinalApplicationStatus(), null, "", 100, appHistory.getApplicationType(), null); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java index 4ba1bdfd1e..0de834c43f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java @@ -391,7 +391,7 @@ public static ApplicationReport newApplicationReport( ApplicationId applicationId, ApplicationAttemptId applicationAttemptId, String user, String queue, String name, String host, int rpcPort, Token clientToAMToken, YarnApplicationState state, String diagnostics, - String url, long startTime, long finishTime, + String url, long startTime, long launchTime, long finishTime, FinalApplicationStatus finalStatus, ApplicationResourceUsageReport appResources, String origTrackingUrl, float progress, String appType, Token amRmToken, Set tags, @@ -410,6 +410,7 @@ public static ApplicationReport newApplicationReport( report.setDiagnostics(diagnostics); report.setTrackingUrl(url); report.setStartTime(startTime); + report.setLaunchTime(launchTime); report.setFinishTime(finishTime); report.setFinalApplicationStatus(finalStatus); report.setApplicationResourceUsageReport(appResources); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java index 8600b7252b..9dd8aa1d24 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java @@ -242,10 +242,9 @@ private void generateOverviewTable(AppInfo app, String schedulerPath, .__("FinalStatus Reported by AM:", clairfyAppFinalStatus(app.getFinalAppStatus())) .__("Started:", Times.format(app.getStartedTime())) - .__( - "Elapsed:", - StringUtils.formatTime(Times.elapsed(app.getStartedTime(), - app.getFinishedTime()))) + .__("Launched:", Times.format(app.getLaunchTime())) + .__("Finished:", Times.format(app.getFinishedTime())) + .__("Elapsed:", StringUtils.formatTime(app.getElapsedTime())) .__( "Tracking URL:", app.getTrackingUrl() == null diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java index b2dd4de818..cb15449d35 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java @@ -150,7 +150,9 @@ protected void renderData(Block html) { html.table("#apps").thead().tr().th(".id", "ID").th(".user", "User") .th(".name", "Name").th(".type", "Application Type") .th(".queue", "Queue").th(".priority", "Application Priority") - .th(".starttime", "StartTime").th(".finishtime", "FinishTime") + .th(".starttime", "StartTime") + .th(".launchtime", "LaunchTime") + .th(".finishtime", "FinishTime") .th(".state", "State").th(".finalstatus", "FinalStatus") .th(".progress", "Progress").th(".ui", "Tracking UI").__().__().tbody(); @@ -188,6 +190,7 @@ protected void renderData(Block html) { .getQueue()))).append("\",\"").append(String .valueOf(app.getPriority())) .append("\",\"").append(app.getStartedTime()) + .append("\",\"").append(app.getLaunchTime()) .append("\",\"").append(app.getFinishedTime()) .append("\",\"") .append(app.getAppState() == null ? UNAVAILABLE : app.getAppState()) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java index 29f1e63456..b2f65a8450 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java @@ -51,7 +51,7 @@ private static String getAppsTableColumnDefs( sb.append("[\n") .append("{'sType':'natural', 'aTargets': [0]") .append(", 'mRender': parseHadoopID }") - .append("\n, {'sType':'numeric', 'aTargets': [6, 7]") + .append("\n, {'sType':'numeric', 'aTargets': [6, 7, 8]") .append(", 'mRender': renderHadoopDate }") .append("\n, {'sType':'numeric', bSearchable:false, 'aTargets':"); if (isFairSchedulerPage) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java index ac2f8da6aa..b4687a4260 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java @@ -55,6 +55,7 @@ public class AppInfo { protected FinalApplicationStatus finalAppStatus; protected long submittedTime; protected long startedTime; + private long launchTime; protected long finishedTime; protected long elapsedTime; protected String applicationTags; @@ -88,6 +89,7 @@ public AppInfo(ApplicationReport app) { originalTrackingUrl = app.getOriginalTrackingUrl(); submittedTime = app.getStartTime(); startedTime = app.getStartTime(); + launchTime = app.getLaunchTime(); finishedTime = app.getFinishTime(); elapsedTime = Times.elapsed(startedTime, finishedTime); finalAppStatus = app.getFinalApplicationStatus(); @@ -198,6 +200,10 @@ public long getSubmittedTime() { return submittedTime; } + public long getLaunchTime() { + return launchTime; + } + public long getStartedTime() { return startedTime; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index 7051f8cca2..58968089f2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -303,7 +303,7 @@ public void run() { LOG.info("Launching master" + application.getAppAttemptId()); launch(); handler.handle(new RMAppAttemptEvent(application.getAppAttemptId(), - RMAppAttemptEventType.LAUNCHED)); + RMAppAttemptEventType.LAUNCHED, System.currentTimeMillis())); } catch(Exception ie) { String message = "Error launching " + application.getAppAttemptId() + ". Got exception: " + StringUtils.stringifyException(ie); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java index 79a5de2584..2b0bd2b5ba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java @@ -47,7 +47,8 @@ public abstract class ApplicationStateData { public static ApplicationStateData newInstance(long submitTime, long startTime, String user, ApplicationSubmissionContext submissionContext, RMAppState state, - String diagnostics, long finishTime, CallerContext callerContext) { + String diagnostics, long launchTime, long finishTime, + CallerContext callerContext) { ApplicationStateData appState = Records.newRecord(ApplicationStateData.class); appState.setSubmitTime(submitTime); appState.setStartTime(startTime); @@ -55,6 +56,7 @@ public static ApplicationStateData newInstance(long submitTime, appState.setApplicationSubmissionContext(submissionContext); appState.setState(state); appState.setDiagnostics(diagnostics); + appState.setLaunchTime(launchTime); appState.setFinishTime(finishTime); appState.setCallerContext(callerContext); return appState; @@ -63,7 +65,8 @@ public static ApplicationStateData newInstance(long submitTime, public static ApplicationStateData newInstance(long submitTime, long startTime, String user, ApplicationSubmissionContext submissionContext, RMAppState state, - String diagnostics, long finishTime, CallerContext callerContext, + String diagnostics, long launchTime, long finishTime, + CallerContext callerContext, Map applicationTimeouts) { ApplicationStateData appState = Records.newRecord(ApplicationStateData.class); @@ -73,6 +76,7 @@ public static ApplicationStateData newInstance(long submitTime, appState.setApplicationSubmissionContext(submissionContext); appState.setState(state); appState.setDiagnostics(diagnostics); + appState.setLaunchTime(launchTime); appState.setFinishTime(finishTime); appState.setCallerContext(callerContext); appState.setApplicationTimeouts(applicationTimeouts); @@ -82,7 +86,7 @@ public static ApplicationStateData newInstance(long submitTime, public static ApplicationStateData newInstance(long submitTime, long startTime, ApplicationSubmissionContext context, String user, CallerContext callerContext) { - return newInstance(submitTime, startTime, user, context, null, "", 0, + return newInstance(submitTime, startTime, user, context, null, "", 0, 0, callerContext); } @@ -136,6 +140,20 @@ public int getFirstAttemptId() { @Unstable public abstract void setStartTime(long startTime); + + + /** + * Get the launch time of the application. + * @return launch time of the application + */ + @Public + @Stable + public abstract long getLaunchTime(); + + @Private + @Unstable + public abstract void setLaunchTime(long launchTime); + /** * The application submitter */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java index d037e68a12..f5cd107e17 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java @@ -120,6 +120,19 @@ public void setStartTime(long startTime) { builder.setStartTime(startTime); } + + @Override + public long getLaunchTime() { + ApplicationStateDataProtoOrBuilder p = viaProto ? proto : builder; + return p.getLaunchTime(); + } + + @Override + public void setLaunchTime(long launchTime) { + maybeInitBuilder(); + builder.setLaunchTime(launchTime); + } + @Override public String getUser() { ApplicationStateDataProtoOrBuilder p = viaProto ? proto : builder; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java index e286834cd4..99cce87102 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java @@ -180,7 +180,15 @@ ApplicationReport createAndGetApplicationReport(String clientUserName, * @return the submit time of the application. */ long getSubmitTime(); - + + /** + * The launch time of the application. + * Since getStartTime() returns what is essentially submit time, + * this new field is to prevent potential backwards compatibility issues. + * @return the launch time of the application. + */ + long getLaunchTime(); + /** * The tracking url for the application master. * @return the tracking url for the application master. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEvent.java index 5c46945754..3212d6c504 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEvent.java @@ -37,6 +37,12 @@ public RMAppEvent(ApplicationId appId, RMAppEventType type, this.diagnosticMsg = diagnostic; } + public RMAppEvent(ApplicationId appId, RMAppEventType type, long timeStamp) { + super(type, timeStamp); + this.appId = appId; + this.diagnosticMsg = ""; + } + public ApplicationId getApplicationId() { return this.appId; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java index 514efd44dd..4b55d38954 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java @@ -37,6 +37,7 @@ public enum RMAppEventType { ATTEMPT_FAILED, ATTEMPT_KILLED, NODE_UPDATE, + ATTEMPT_LAUNCHED, // Source: Container and ResourceTracker APP_RUNNING_ON_NODE, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 4234e58257..daf14c4663 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -164,6 +164,7 @@ public class RMAppImpl implements RMApp, Recoverable { // Mutable fields private long startTime; + private long launchTime = 0; private long finishTime = 0; private long storedFinishTime = 0; private int firstAttemptIdInStateStore = 1; @@ -290,6 +291,10 @@ RMAppEventType.KILL, new KillAttemptTransition()) .addTransition(RMAppState.ACCEPTED, RMAppState.ACCEPTED, RMAppEventType.APP_RUNNING_ON_NODE, new AppRunningOnNodeTransition()) + // Handle AppAttemptLaunch to upate the launchTime and publish to ATS + .addTransition(RMAppState.ACCEPTED, RMAppState.ACCEPTED, + RMAppEventType.ATTEMPT_LAUNCHED, + new AttemptLaunchedTransition()) // Transitions from RUNNING state .addTransition(RMAppState.RUNNING, RMAppState.RUNNING, @@ -784,9 +789,9 @@ public ApplicationReport createAndGetApplicationReport(String clientUserName, this.applicationId, currentApplicationAttemptId, this.user, this.queue, this.name, host, rpcPort, clientToAMToken, createApplicationState(), diags, trackingUrl, this.startTime, - this.finishTime, finishState, appUsageReport, origTrackingUrl, - progress, this.applicationType, amrmToken, applicationTags, - this.getApplicationPriority()); + this.launchTime, this.finishTime, finishState, appUsageReport, + origTrackingUrl, progress, this.applicationType, amrmToken, + applicationTags, this.getApplicationPriority()); report.setLogAggregationStatus(logAggregationStatus); report.setUnmanagedApp(submissionContext.getUnmanagedAM()); report.setAppNodeLabelExpression(getAppNodeLabelExpression()); @@ -839,6 +844,17 @@ public long getStartTime() { } } + @Override + public long getLaunchTime() { + this.readLock.lock(); + + try { + return this.launchTime; + } finally { + this.readLock.unlock(); + } + } + @Override public long getSubmitTime() { return this.submitTime; @@ -936,6 +952,7 @@ public void recover(RMState state) { .getDiagnostics()); this.storedFinishTime = appState.getFinishTime(); this.startTime = appState.getStartTime(); + this.launchTime = appState.getLaunchTime(); this.callerContext = appState.getCallerContext(); this.applicationTimeouts = appState.getApplicationTimeouts(); // If interval > 0, some attempts might have been deleted. @@ -1038,6 +1055,21 @@ public void transition(RMAppImpl app, RMAppEvent event) { }; } + private static final class AttemptLaunchedTransition + extends RMAppTransition { + @Override + public void transition(RMAppImpl app, RMAppEvent event) { + + if(app.launchTime == 0) { + LOG.info("update the launch time for applicationId: "+ + app.getApplicationId()+", attemptId: "+ + app.getCurrentAppAttempt().getAppAttemptId()+ + "launchTime: "+event.getTimestamp()); + app.launchTime = event.getTimestamp(); + } + } + } + private static final class AppRunningOnNodeTransition extends RMAppTransition { public void transition(RMAppImpl app, RMAppEvent event) { RMAppRunningOnNodeEvent nodeAddedEvent = (RMAppRunningOnNodeEvent) event; @@ -1297,7 +1329,8 @@ private void rememberTargetTransitionsAndStoreState(RMAppEvent event, ApplicationStateData appState = ApplicationStateData.newInstance(this.submitTime, this.startTime, this.user, this.submissionContext, - stateToBeStored, diags, this.storedFinishTime, this.callerContext); + stateToBeStored, diags, this.launchTime, this.storedFinishTime, + this.callerContext); appState.setApplicationTimeouts(this.applicationTimeouts); this.rmContext.getStateStore().updateApplicationState(appState); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptEvent.java index 6df6b19f97..2ddc981067 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptEvent.java @@ -38,6 +38,13 @@ public RMAppAttemptEvent(ApplicationAttemptId appAttemptId, this.diagnosticMsg = diagnostics; } + public RMAppAttemptEvent(ApplicationAttemptId appAttemptId, + RMAppAttemptEventType type, long timeStamp) { + super(type, timeStamp); + this.appAttemptId = appAttemptId; + this.diagnosticMsg = ""; + } + public ApplicationAttemptId getApplicationAttemptId() { return this.appAttemptId; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 1b1e2c447c..32f275f807 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -1557,7 +1557,9 @@ public void transition(RMAppAttemptImpl appAttempt, appAttempt.launchAMStartTime; ClusterMetrics.getMetrics().addAMLaunchDelay(delay); } - + appAttempt.eventHandler.handle( + new RMAppEvent(appAttempt.getAppAttemptId().getApplicationId(), + RMAppEventType.ATTEMPT_LAUNCHED, event.getTimestamp())); appAttempt .updateAMLaunchDiagnostics(AMState.LAUNCHED.getDiagnosticMessage()); // Register with AMLivelinessMonitor diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java index ac88f861fe..2bbbcc54a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java @@ -89,6 +89,7 @@ public FairSchedulerAppsBlock(ResourceManager rm, ViewContext ctx, th(".queue", "Queue"). th(".fairshare", "Fair Share"). th(".starttime", "StartTime"). + th(".launchTime", "LaunchTime"). th(".finishtime", "FinishTime"). th(".state", "State"). th(".finalstatus", "FinalStatus"). @@ -135,6 +136,7 @@ public FairSchedulerAppsBlock(ResourceManager rm, ViewContext ctx, appInfo.getQueue()))).append("\",\"") .append(fairShare).append("\",\"") .append(appInfo.getStartTime()).append("\",\"") + .append(appInfo.getLaunchTime()).append("\",\"") .append(appInfo.getFinishTime()).append("\",\"") .append(appInfo.getState()).append("\",\"") .append(appInfo.getFinalStatus()).append("\",\"") diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java index d0dccab6e5..a525cffdd0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java @@ -62,6 +62,7 @@ protected void renderData(Block html) { .th(".name", "Name").th(".type", "Application Type") .th(".queue", "Queue").th(".priority", "Application Priority") .th(".starttime", "StartTime") + .th("launchtime", "LaunchTime") .th(".finishtime", "FinishTime").th(".state", "State") .th(".finalstatus", "FinalStatus") .th(".runningcontainer", "Running Containers") @@ -134,6 +135,7 @@ protected void renderData(Block html) { .getQueue()))).append("\",\"").append(String .valueOf(app.getPriority())) .append("\",\"").append(app.getStartedTime()) + .append("\",\"").append(app.getLaunchTime()) .append("\",\"").append(app.getFinishedTime()) .append("\",\"") .append(app.getAppState() == null ? UNAVAILABLE : app.getAppState()) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 631c908419..d47f13d958 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -87,6 +87,7 @@ public class AppInfo { // these are only allowed if acls allow protected long startedTime; + private long launchTime; protected long finishedTime; protected long elapsedTime; protected String amContainerLogs; @@ -181,6 +182,7 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, this.clusterId = ResourceManager.getClusterTimeStamp(); if (hasAccess) { this.startedTime = app.getStartTime(); + this.launchTime = app.getLaunchTime(); this.finishedTime = app.getFinishTime(); this.elapsedTime = Times.elapsed(app.getStartTime(), app.getFinishTime()); @@ -394,6 +396,10 @@ public long getStartTime() { return this.startedTime; } + public long getLaunchTime() { + return this.launchTime; + } + public long getFinishTime() { return this.finishedTime; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto index 39a56a811a..35c77ab78b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto @@ -70,6 +70,7 @@ message ApplicationStateDataProto { optional int64 finish_time = 7; optional hadoop.common.RPCCallerContextProto caller_context = 8; repeated ApplicationTimeoutMapProto application_timeouts = 9; + optional int64 launch_time = 10; } message ApplicationAttemptStateDataProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index 72de27cf95..6c6c4b4e80 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -92,7 +92,12 @@ public long getStartTime() { public long getSubmitTime() { throw new UnsupportedOperationException("Not supported yet."); } - + + @Override + public long getLaunchTime() { + throw new UnsupportedOperationException("Not supported yet."); + } + @Override public long getFinishTime() { throw new UnsupportedOperationException("Not supported yet."); @@ -272,6 +277,7 @@ public static RMApp newApplication(int i) { final String name = newAppName(); final String queue = newQueue(); final long start = 123456 + i * 1000; + final long launch = start + i * 100; final long finish = 234567 + i * 1000; final String type = YarnConfiguration.DEFAULT_APPLICATION_TYPE; YarnApplicationState[] allStates = YarnApplicationState.values(); @@ -307,6 +313,11 @@ public long getStartTime() { return start; } + @Override + public long getLaunchTime() { + return launch; + } + @Override public long getFinishTime() { return finish; @@ -357,7 +368,7 @@ public ApplicationReport createAndGetApplicationReport( ApplicationReport report = ApplicationReport.newInstance( getApplicationId(), appAttemptId, getUser(), getQueue(), getName(), null, 0, null, null, getDiagnostics().toString(), - getTrackingUrl(), getStartTime(), getFinishTime(), + getTrackingUrl(), getLaunchTime(), getStartTime(), getFinishTime(), getFinalApplicationStatus(), usageReport , null, getProgress(), type, null); return report; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java index dbb214897a..957d4ce822 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java @@ -358,7 +358,7 @@ void testRMAppStateStore(RMStateStoreHelper stateStoreHelper, ApplicationStateData.newInstance(appState.getSubmitTime(), appState.getStartTime(), appState.getUser(), appState.getApplicationSubmissionContext(), RMAppState.FINISHED, - "appDiagnostics", 1234, appState.getCallerContext()); + "appDiagnostics", 123, 1234, appState.getCallerContext()); appState2.attempts.putAll(appState.attempts); store.updateApplicationState(appState2); @@ -384,7 +384,7 @@ void testRMAppStateStore(RMStateStoreHelper stateStoreHelper, ApplicationStateData dummyApp = ApplicationStateData.newInstance(appState.getSubmitTime(), appState.getStartTime(), appState.getUser(), dummyContext, - RMAppState.FINISHED, "appDiagnostics", 1234, null); + RMAppState.FINISHED, "appDiagnostics", 123, 1234, null); store.updateApplicationState(dummyApp); ApplicationAttemptId dummyAttemptId = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java index 0738730aab..fe4a7019ba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java @@ -407,7 +407,7 @@ public void run() { store.storeApplicationStateInternal( ApplicationId.newInstance(100L, 1), ApplicationStateData.newInstance(111, 111, "user", null, - RMAppState.ACCEPTED, "diagnostics", 333, null)); + RMAppState.ACCEPTED, "diagnostics", 222, 333, null)); } catch (Exception e) { assertionFailedInThread.set(true); e.printStackTrace(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java index 0a1b15275c..d8718e0b88 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java @@ -790,7 +790,7 @@ private static ApplicationStateData createAppState( long finishTime, boolean isFinished) { return ApplicationStateData.newInstance(submitTime, startTime, "test", ctxt, isFinished ? RMAppState.FINISHED : null, isFinished ? - "appDiagnostics" : "", isFinished ? finishTime : 0, null); + "appDiagnostics" : "", 0, isFinished ? finishTime : 0, null); } private static ApplicationAttemptStateData createFinishedAttempt( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java index 664fae2d06..ad29d274a4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java @@ -57,6 +57,7 @@ public class MockRMApp implements RMApp { String name = MockApps.newAppName(); String queue = MockApps.newQueue(); long start = System.currentTimeMillis() - (int) (Math.random() * DT); + private long launch = start; long submit = start - (int) (Math.random() * DT); long finish = 0; RMAppState state = RMAppState.NEW; @@ -194,6 +195,11 @@ public long getSubmitTime() { return submit; } + @Override + public long getLaunchTime() { + return launch; + } + public void setStartTime(long time) { this.start = time; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index 920bd4fd2b..70887e0f4b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -1239,7 +1239,7 @@ public void createRMStateForApplications( ApplicationStateData appState = ApplicationStateData.newInstance(app.getSubmitTime(), app.getStartTime(), app.getUser(), app.getApplicationSubmissionContext(), rmAppState, - null, app.getFinishTime(), null); + null, app.getLaunchTime(), app.getFinishTime(), null); applicationState.put(app.getApplicationId(), appState); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java index b1256083ef..93377be1ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java @@ -257,7 +257,7 @@ public static ClientRMService mockClientRMService(RMContext rmContext) { app.getName(), (String) null, 0, (Token) null, app.createApplicationState(), app.getDiagnostics().toString(), (String) null, - app.getStartTime(), app.getFinishTime(), + app.getStartTime(), app.getLaunchTime(), app.getFinishTime(), app.getFinalApplicationStatus(), (ApplicationResourceUsageReport) null, app.getTrackingUrl(), app.getProgress(), app.getApplicationType(), (Token) null); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index a381ed4020..6c6f400a62 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1553,6 +1553,7 @@ public void verifyAppsXML(NodeList nodes, RMApp app, boolean hasResourceReq) WebServicesTestUtils.getXmlString(element, "diagnostics"), WebServicesTestUtils.getXmlLong(element, "clusterId"), WebServicesTestUtils.getXmlLong(element, "startedTime"), + WebServicesTestUtils.getXmlLong(element, "launchTime"), WebServicesTestUtils.getXmlLong(element, "finishedTime"), WebServicesTestUtils.getXmlLong(element, "elapsedTime"), WebServicesTestUtils.getXmlString(element, "amHostHttpAddress"), @@ -1603,7 +1604,7 @@ public void verifyAppsXML(NodeList nodes, RMApp app, boolean hasResourceReq) public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs) throws JSONException, Exception { - int expectedNumberOfElements = 39 + (hasResourceReqs ? 2 : 0); + int expectedNumberOfElements = 40 + (hasResourceReqs ? 2 : 0); String appNodeLabelExpression = null; String amNodeLabelExpression = null; if (app.getApplicationSubmissionContext() @@ -1629,8 +1630,10 @@ public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs) info.getString("state"), info.getString("finalStatus"), (float) info.getDouble("progress"), info.getString("trackingUI"), info.getString("diagnostics"), info.getLong("clusterId"), - info.getLong("startedTime"), info.getLong("finishedTime"), - info.getLong("elapsedTime"), info.getString("amHostHttpAddress"), + info.getLong("startedTime"), info.getLong("launchTime"), + info.getLong("finishedTime"), + info.getLong("elapsedTime"), + info.getString("amHostHttpAddress"), info.getString("amContainerLogs"), info.getInt("allocatedMB"), info.getInt("allocatedVCores"), info.getInt("runningContainers"), (float) info.getDouble("queueUsagePercentage"), @@ -1653,8 +1656,9 @@ public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs) public void verifyAppInfoGeneric(RMApp app, String id, String user, String name, String applicationType, String queue, int prioirty, String state, String finalStatus, float progress, String trackingUI, - String diagnostics, long clusterId, long startedTime, long finishedTime, - long elapsedTime, String amHostHttpAddress, String amContainerLogs, + String diagnostics, long clusterId, long startedTime, + long launchTime, long finishedTime, long elapsedTime, + String amHostHttpAddress, String amContainerLogs, int allocatedMB, int allocatedVCores, int numContainers, float queueUsagePerc, float clusterUsagePerc, int preemptedResourceMB, int preemptedResourceVCores,