From 7fa62e150ce86f7cb0e88d593b8f07e3384a9a60 Mon Sep 17 00:00:00 2001 From: Rohith Sharma K S Date: Wed, 6 Feb 2019 09:31:20 +0530 Subject: [PATCH] YARN-8219. Add application launch time to ATSV2. Contributed by Abhishek Modi. --- .../metrics/ApplicationMetricsConstants.java | 3 +++ .../util/timeline/TimelineEntityV2Converter.java | 9 +++++++-- .../metrics/AbstractSystemMetricsPublisher.java | 5 +++++ .../metrics/CombinedSystemMetricsPublisher.java | 6 ++++++ .../metrics/NoOpSystemMetricPublisher.java | 4 ++++ .../metrics/SystemMetricsPublisher.java | 2 ++ .../metrics/TimelineServiceV2Publisher.java | 14 ++++++++++++++ .../server/resourcemanager/rmapp/RMAppImpl.java | 4 +++- .../metrics/TestSystemMetricsPublisherForV2.java | 4 +++- 9 files changed, 47 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java index 93da565341..6428fa6768 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java @@ -34,6 +34,9 @@ public class ApplicationMetricsConstants { public static final String FINISHED_EVENT_TYPE = "YARN_APPLICATION_FINISHED"; + public static final String LAUNCHED_EVENT_TYPE = + "YARN_APPLICATION_LAUNCHED"; + public static final String ACLS_UPDATED_EVENT_TYPE = "YARN_APPLICATION_ACLS_UPDATED"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineEntityV2Converter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineEntityV2Converter.java index 98e1e7a7f4..9fefc09bc0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineEntityV2Converter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineEntityV2Converter.java @@ -258,6 +258,7 @@ public static ApplicationReport convertToApplicationReport( String type = null; boolean unmanagedApplication = false; long createdTime = 0; + long launchTime = 0; long finishedTime = 0; float progress = 0.0f; int applicationPriority = 0; @@ -416,6 +417,9 @@ public static ApplicationReport convertToApplicationReport( if (event.getId().equals( ApplicationMetricsConstants.CREATED_EVENT_TYPE)) { createdTime = event.getTimestamp(); + } else if (event.getId().equals( + ApplicationMetricsConstants.LAUNCHED_EVENT_TYPE)) { + launchTime = event.getTimestamp(); } else if (event.getId().equals( ApplicationMetricsConstants.UPDATED_EVENT_TYPE)) { // This type of events are parsed in time-stamp descending order @@ -449,8 +453,9 @@ public static ApplicationReport convertToApplicationReport( return ApplicationReport.newInstance( ApplicationId.fromString(entity.getId()), latestApplicationAttemptId, user, queue, name, null, -1, null, state, - diagnosticsInfo, null, createdTime, finishedTime, finalStatus, - appResources, null, progress, type, null, appTags, unmanagedApplication, + diagnosticsInfo, null, createdTime, launchTime, + finishedTime, finalStatus, appResources, null, + progress, type, null, appTags, unmanagedApplication, Priority.newInstance(applicationPriority), appNodeLabelExpression, amNodeLabelExpression); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractSystemMetricsPublisher.java index 96b153f85c..09f44d06f8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractSystemMetricsPublisher.java @@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.Event; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; /** * Abstract implementation of SystemMetricsPublisher which is then extended by @@ -122,6 +123,10 @@ protected enum SystemMetricsEventType { PUBLISH_ENTITY, PUBLISH_APPLICATION_FINISHED_ENTITY } + @Override + public void appLaunched(RMApp app, long launchTime) { + } + /** * TimelinePublishEvent's hash code should be based on application's id this * will ensure all the events related to a particular app goes to particular diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/CombinedSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/CombinedSystemMetricsPublisher.java index 9646747798..8e04a1ecd7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/CombinedSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/CombinedSystemMetricsPublisher.java @@ -45,6 +45,12 @@ public void appCreated(RMApp app, long createdTime) { } } + @Override public void appLaunched(RMApp app, long launchTime) { + for (SystemMetricsPublisher publisher : this.publishers) { + publisher.appLaunched(app, launchTime); + } + } + @Override public void appACLsUpdated(RMApp app, String appViewACLs, long updatedTime) { for (SystemMetricsPublisher publisher : this.publishers) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/NoOpSystemMetricPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/NoOpSystemMetricPublisher.java index e0bf3c7c5c..537c88cccd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/NoOpSystemMetricPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/NoOpSystemMetricPublisher.java @@ -67,4 +67,8 @@ public void appUpdated(RMApp app, long currentTimeMillis) { public void appStateUpdated(RMApp app, YarnApplicationState appState, long updatedTime) { } + + @Override + public void appLaunched(RMApp app, long launchTime) { + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java index eeea4344cf..137d7903b5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java @@ -32,6 +32,8 @@ public interface SystemMetricsPublisher { void appCreated(RMApp app, long createdTime); + void appLaunched(RMApp app, long launchTime); + void appACLsUpdated(RMApp app, String appViewACLs, long updatedTime); void appUpdated(RMApp app, long updatedTime); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java index 95d75b2306..e93c26d6b9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java @@ -153,6 +153,20 @@ public void appCreated(RMApp app, long createdTime) { SystemMetricsEventType.PUBLISH_ENTITY, entity, app.getApplicationId())); } + @SuppressWarnings("unchecked") + @Override + public void appLaunched(RMApp app, long launchTime) { + ApplicationEntity entity = + createApplicationEntity(app.getApplicationId()); + TimelineEvent tEvent = new TimelineEvent(); + tEvent.setId(ApplicationMetricsConstants.LAUNCHED_EVENT_TYPE); + tEvent.setTimestamp(launchTime); + entity.addEvent(tEvent); + + getDispatcher().getEventHandler().handle(new TimelineV2PublishEvent( + SystemMetricsEventType.PUBLISH_ENTITY, entity, app.getApplicationId())); + } + @SuppressWarnings("unchecked") @Override public void appFinished(RMApp app, RMAppState state, long finishedTime) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index bd645d7c73..696b39d001 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -293,7 +293,7 @@ RMAppEventType.KILL, new KillAttemptTransition()) .addTransition(RMAppState.ACCEPTED, RMAppState.ACCEPTED, RMAppEventType.APP_RUNNING_ON_NODE, new AppRunningOnNodeTransition()) - // Handle AppAttemptLaunch to upate the launchTime and publish to ATS + // Handle AppAttemptLaunch to update the launchTime and publish to ATS .addTransition(RMAppState.ACCEPTED, RMAppState.ACCEPTED, RMAppEventType.ATTEMPT_LAUNCHED, new AttemptLaunchedTransition()) @@ -1067,6 +1067,8 @@ public void transition(RMAppImpl app, RMAppEvent event) { app.getCurrentAppAttempt().getAppAttemptId()+ "launchTime: "+event.getTimestamp()); app.launchTime = event.getTimestamp(); + app.rmContext.getSystemMetricsPublisher().appLaunched( + app, app.launchTime); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java index 3b503e54a4..ca3f47df24 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java @@ -203,6 +203,7 @@ public void testPublishApplicationMetrics() throws Exception { RMApp app = createAppAndRegister(appId); metricsPublisher.appCreated(app, app.getStartTime()); + metricsPublisher.appLaunched(app, app.getLaunchTime()); metricsPublisher.appACLsUpdated(app, "user1,user2", 4L); metricsPublisher.appFinished(app, RMAppState.FINISHED, app.getFinishTime()); dispatcher.await(); @@ -221,7 +222,7 @@ public void testPublishApplicationMetrics() throws Exception { File appFile = new File(outputDirApp, timelineServiceFileName); Assert.assertTrue(appFile.exists()); verifyEntity( - appFile, 3, ApplicationMetricsConstants.CREATED_EVENT_TYPE, 8, 0); + appFile, 4, ApplicationMetricsConstants.CREATED_EVENT_TYPE, 8, 0); } @Test(timeout = 10000) @@ -356,6 +357,7 @@ private static RMApp createRMApp(ApplicationId appId) { when(app.getQueue()).thenReturn("test queue"); when(app.getSubmitTime()).thenReturn(Integer.MAX_VALUE + 1L); when(app.getStartTime()).thenReturn(Integer.MAX_VALUE + 2L); + when(app.getLaunchTime()).thenReturn(Integer.MAX_VALUE + 2L); when(app.getFinishTime()).thenReturn(Integer.MAX_VALUE + 3L); when(app.getDiagnostics()).thenReturn( new StringBuilder("test diagnostics info"));