big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-8218 Add application launch time to ATSV1. Contributed by Abhishek Modi

Browse Source
This commit is contained in:
Vrushali C 2019-03-06 21:47:29 -08:00
parent 09a9938db7
commit 491313ab84
3 changed files with 28 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
View File

@ -250,6 +250,7 @@ private static ApplicationReportExt convertToApplicationReport(
String type = null;
boolean unmanagedApplication = false;
long createdTime = 0;
long launchTime = 0;
long submittedTime = 0;
long finishedTime = 0;
float progress = 0.0f;
@ -378,6 +379,9 @@ private static ApplicationReportExt convertToApplicationReport(
if (event.getEventType().equals(
ApplicationMetricsConstants.CREATED_EVENT_TYPE)) {
createdTime = event.getTimestamp();
} else if (event.getEventType().equals(
ApplicationMetricsConstants.LAUNCHED_EVENT_TYPE)) {
launchTime = event.getTimestamp();
} else if (event.getEventType().equals(
ApplicationMetricsConstants.UPDATED_EVENT_TYPE)) {
// This type of events are parsed in time-stamp descending order
@ -454,7 +458,8 @@ private static ApplicationReportExt convertToApplicationReport(
return new ApplicationReportExt(ApplicationReport.newInstance(
ApplicationId.fromString(entity.getEntityId()),
latestApplicationAttemptId, user, queue, name, null, -1, null, state,
diagnosticsInfo, null, createdTime, submittedTime, 0, finishedTime,
diagnosticsInfo, null, createdTime,
submittedTime, launchTime, finishedTime,
finalStatus, appResources, null, progress, type, null, appTags,
unmanagedApplication, Priority.newInstance(applicationPriority),
appNodeLabelExpression, amNodeLabelExpression), appViewACLs);

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
View File

@ -121,6 +121,19 @@ public void appCreated(RMApp app, long createdTime) {
SystemMetricsEventType.PUBLISH_ENTITY, entity, app.getApplicationId()));
}
@Override
public void appLaunched(RMApp app, long launchTime) {
TimelineEntity entity = createApplicationEntity(app.getApplicationId());
TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType(ApplicationMetricsConstants.LAUNCHED_EVENT_TYPE);
tEvent.setTimestamp(launchTime);
entity.addEvent(tEvent);
getDispatcher().getEventHandler().handle(new TimelineV1PublishEvent(
SystemMetricsEventType.PUBLISH_ENTITY, entity, app.getApplicationId()));
}
@Override
public void appFinished(RMApp app, RMAppState state, long finishedTime) {
TimelineEntity entity = createApplicationEntity(app.getApplicationId());

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
View File

@ -115,6 +115,7 @@ public void testPublishApplicationMetrics() throws Exception {
ApplicationId appId = ApplicationId.newInstance(0, i);
RMApp app = createRMApp(appId);
metricsPublisher.appCreated(app, app.getStartTime());
metricsPublisher.appLaunched(app, app.getLaunchTime());
if (i == 1) {
when(app.getQueue()).thenReturn("new test queue");
ApplicationSubmissionContext asc = mock(
@ -150,7 +151,7 @@ public void testPublishApplicationMetrics() throws Exception {
ApplicationMetricsConstants.ENTITY_TYPE,
EnumSet.allOf(Field.class));
// ensure Five events are both published before leaving the loop
} while (entity == null || entity.getEvents().size() < 5);
} while (entity == null || entity.getEvents().size() < 6);
// verify all the fields
Assert.assertEquals(ApplicationMetricsConstants.ENTITY_TYPE,
entity.getEntityType());
@ -240,6 +241,7 @@ public void testPublishApplicationMetrics() throws Exception {
Assert.assertEquals("context", entity.getOtherInfo()
.get(ApplicationMetricsConstants.YARN_APP_CALLER_CONTEXT));
boolean hasCreatedEvent = false;
boolean hasLaunchedEvent = false;
boolean hasUpdatedEvent = false;
boolean hasFinishedEvent = false;
boolean hasACLsUpdatedEvent = false;
@ -249,6 +251,10 @@ public void testPublishApplicationMetrics() throws Exception {
ApplicationMetricsConstants.CREATED_EVENT_TYPE)) {
hasCreatedEvent = true;
Assert.assertEquals(app.getStartTime(), event.getTimestamp());
} else if (event.getEventType().equals(
ApplicationMetricsConstants.LAUNCHED_EVENT_TYPE)) {
hasLaunchedEvent = true;
Assert.assertEquals(app.getLaunchTime(), event.getTimestamp());
} else if (event.getEventType().equals(
ApplicationMetricsConstants.FINISHED_EVENT_TYPE)) {
hasFinishedEvent = true;
@ -292,6 +298,7 @@ public void testPublishApplicationMetrics() throws Exception {
}
// Do assertTrue verification separately for easier debug
Assert.assertTrue(hasCreatedEvent);
Assert.assertTrue(hasLaunchedEvent);
Assert.assertTrue(hasFinishedEvent);
Assert.assertTrue(hasACLsUpdatedEvent);
Assert.assertTrue(hasUpdatedEvent);
@ -499,6 +506,7 @@ private static RMApp createRMApp(ApplicationId appId) {
when(app.getQueue()).thenReturn("test queue");
when(app.getSubmitTime()).thenReturn(Integer.MAX_VALUE + 1L);
when(app.getStartTime()).thenReturn(Integer.MAX_VALUE + 2L);
when(app.getLaunchTime()).thenReturn(Integer.MAX_VALUE + 2L);
when(app.getFinishTime()).thenReturn(Integer.MAX_VALUE + 3L);
when(app.getDiagnostics()).thenReturn(
new StringBuilder("test diagnostics info"));