From 3e000a919fede85230fcfb06309a1f1d5e0c479c Mon Sep 17 00:00:00 2001 From: Xuan Date: Fri, 5 Jun 2015 12:48:52 -0700 Subject: [PATCH] YARN-1462. AHS API and other AHS changes to handle tags for completed MR jobs. Contributed by Xuan Gong --- hadoop-yarn-project/CHANGES.txt | 2 + .../yarn/api/records/ApplicationReport.java | 19 ++++++++++ ...licationHistoryManagerOnTimelineStore.java | 19 +++++++++- ...licationHistoryManagerOnTimelineStore.java | 9 +++++ .../metrics/ApplicationMetricsConstants.java | 1 + .../metrics/ApplicationCreatedEvent.java | 10 ++++- .../metrics/SystemMetricsPublisher.java | 4 +- .../metrics/TestSystemMetricsPublisher.java | 37 +++++++++++++++++++ .../src/site/markdown/TimelineServer.md | 30 +++++++++++++-- 9 files changed, 124 insertions(+), 7 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index d802703e5c..cc28977a09 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -292,6 +292,8 @@ Release 2.8.0 - UNRELEASED YARN-2392. Add more diags about app retry limits on AM failures. (Steve Loughran via jianhe) + YARN-1462. AHS API and other AHS changes to handle tags for completed MR jobs. (xgong) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java index e5d7254565..444a202e12 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java @@ -83,6 +83,25 @@ public static ApplicationReport newInstance(ApplicationId applicationId, return report; } + @Private + @Unstable + public static ApplicationReport newInstance(ApplicationId applicationId, + ApplicationAttemptId applicationAttemptId, String user, String queue, + String name, String host, int rpcPort, Token clientToAMToken, + YarnApplicationState state, String diagnostics, String url, + long startTime, long finishTime, FinalApplicationStatus finalStatus, + ApplicationResourceUsageReport appResources, String origTrackingUrl, + float progress, String applicationType, Token amRmToken, + Set tags) { + ApplicationReport report = + newInstance(applicationId, applicationAttemptId, user, queue, name, + host, rpcPort, clientToAMToken, state, diagnostics, url, startTime, + finishTime, finalStatus, appResources, origTrackingUrl, progress, + applicationType, amRmToken); + report.setApplicationTags(tags); + return report; + } + /** * Get the ApplicationId of the application. * @return ApplicationId of the application diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index 9bfd2d63b1..0c7fdc0a9b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -19,11 +19,14 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice; import java.io.IOException; +import java.util.Collection; import java.util.EnumSet; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -249,6 +252,7 @@ private static ApplicationReportExt convertToApplicationReport( FinalApplicationStatus finalStatus = FinalApplicationStatus.UNDEFINED; YarnApplicationState state = YarnApplicationState.ACCEPTED; ApplicationResourceUsageReport appResources = null; + Set appTags = null; Map appViewACLs = new HashMap(); Map entityInfo = entity.getOtherInfo(); @@ -270,7 +274,7 @@ private static ApplicationReportExt convertToApplicationReport( ConverterUtils.toApplicationId(entity.getEntityId()), latestApplicationAttemptId, user, queue, name, null, -1, null, state, diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null, - null, progress, type, null), appViewACLs); + null, progress, type, null, appTags), appViewACLs); } if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) { queue = @@ -295,6 +299,17 @@ private static ApplicationReportExt convertToApplicationReport( appResources=ApplicationResourceUsageReport .newInstance(0, 0, null, null, null, memorySeconds, vcoreSeconds); } + if (entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) { + appTags = new HashSet(); + Object obj = entityInfo.get(ApplicationMetricsConstants.APP_TAGS_INFO); + if (obj != null && obj instanceof Collection) { + for(Object o : (Collection)obj) { + if (o != null) { + appTags.add(o.toString()); + } + } + } + } } List events = entity.getEvents(); if (events != null) { @@ -347,7 +362,7 @@ private static ApplicationReportExt convertToApplicationReport( ConverterUtils.toApplicationId(entity.getEntityId()), latestApplicationAttemptId, user, queue, name, null, -1, null, state, diagnosticsInfo, null, createdTime, finishedTime, finalStatus, appResources, - null, progress, type, null), appViewACLs); + null, progress, type, null, appTags), appViewACLs); } private static ApplicationAttemptReport convertToApplicationAttemptReport( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java index 86729536cd..dbd75acfea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java @@ -22,7 +22,9 @@ import java.util.Arrays; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.SaslRpcServer.AuthMethod; @@ -185,6 +187,9 @@ public ApplicationReport run() throws Exception { Assert.assertEquals(Integer.MAX_VALUE + 3L + +app.getApplicationId().getId(), app.getFinishTime()); Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001); + Assert.assertEquals(2, app.getApplicationTags().size()); + Assert.assertTrue(app.getApplicationTags().contains("Test_APP_TAGS_1")); + Assert.assertTrue(app.getApplicationTags().contains("Test_APP_TAGS_2")); // App 2 doesn't have the ACLs, such that the default ACLs " " will be used. // Nobody except admin and owner has access to the details of the app. if ((i == 1 && callerUGI != null && @@ -471,6 +476,10 @@ private static TimelineEntity createApplicationTimelineEntity( entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, "user2"); } + Set appTags = new HashSet(); + appTags.add("Test_APP_TAGS_1"); + appTags.add("Test_APP_TAGS_2"); + entityInfo.put(ApplicationMetricsConstants.APP_TAGS_INFO, appTags); entity.setOtherInfo(entityInfo); TimelineEvent tEvent = new TimelineEvent(); tEvent.setEventType(ApplicationMetricsConstants.CREATED_EVENT_TYPE); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java index df8eecb40b..f452410617 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java @@ -73,4 +73,5 @@ public class ApplicationMetricsConstants { public static final String LATEST_APP_ATTEMPT_EVENT_INFO = "YARN_APPLICATION_LATEST_APP_ATTEMPT"; + public static final String APP_TAGS_INFO = "YARN_APPLICATION_TAGS"; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java index 2373b3bf1e..7c43aa4ba6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.metrics; +import java.util.Set; + import org.apache.hadoop.yarn.api.records.ApplicationId; public class ApplicationCreatedEvent extends @@ -29,6 +31,7 @@ public class ApplicationCreatedEvent extends private String user; private String queue; private long submittedTime; + private Set appTags; public ApplicationCreatedEvent(ApplicationId appId, String name, @@ -36,7 +39,8 @@ public ApplicationCreatedEvent(ApplicationId appId, String user, String queue, long submittedTime, - long createdTime) { + long createdTime, + Set appTags) { super(SystemMetricsEventType.APP_CREATED, createdTime); this.appId = appId; this.name = name; @@ -44,6 +48,7 @@ public ApplicationCreatedEvent(ApplicationId appId, this.user = user; this.queue = queue; this.submittedTime = submittedTime; + this.appTags = appTags; } @Override @@ -75,4 +80,7 @@ public long getSubmittedTime() { return submittedTime; } + public Set getAppTags() { + return appTags; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java index b849b0035d..faf6d7265f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java @@ -106,7 +106,7 @@ public void appCreated(RMApp app, long createdTime) { app.getUser(), app.getQueue(), app.getSubmitTime(), - createdTime)); + createdTime, app.getApplicationTags())); } } @@ -251,6 +251,8 @@ private void publishApplicationCreatedEvent(ApplicationCreatedEvent event) { event.getQueue()); entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO, event.getSubmittedTime()); + entityInfo.put(ApplicationMetricsConstants.APP_TAGS_INFO, + event.getAppTags()); entity.setOtherInfo(entityInfo); TimelineEvent tEvent = new TimelineEvent(); tEvent.setEventType( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java index 7ed3835bdb..5197902a9b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -21,7 +21,11 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import java.util.Collection; import java.util.EnumSet; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -143,6 +147,8 @@ public void testPublishApplicationMetrics() throws Exception { Assert.assertEquals(app.getSubmitTime(), entity.getOtherInfo().get( ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO)); + Assert.assertTrue(verifyAppTags(app.getApplicationTags(), + entity.getOtherInfo())); if (i == 1) { Assert.assertEquals("uers1,user2", entity.getOtherInfo().get( @@ -352,6 +358,10 @@ private static RMApp createRMApp(ApplicationId appId) { FinalApplicationStatus.UNDEFINED); when(app.getRMAppMetrics()).thenReturn( new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE)); + Set appTags = new HashSet(); + appTags.add("test"); + appTags.add("tags"); + when(app.getApplicationTags()).thenReturn(appTags); return app; } @@ -392,4 +402,31 @@ private static RMContainer createRMContainer(ContainerId containerId) { return container; } + private static boolean verifyAppTags(Set appTags, + Map entityInfo) { + if (!entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) { + return false; + } + Object obj = entityInfo.get(ApplicationMetricsConstants.APP_TAGS_INFO); + if (obj instanceof Collection) { + Collection collection = (Collection) obj; + if (collection.size() != appTags.size()) { + return false; + } + for (String appTag : appTags) { + boolean match = false; + for (Object o : collection) { + if (o.toString().equals(appTag)) { + match = true; + break; + } + } + if (!match) { + return false; + } + } + return true; + } + return false; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md index 90f9512981..1b666068a9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md @@ -1125,7 +1125,27 @@ Response Body: "submittedTime":1430424769395, "startedTime":1430424769395, "finishedTime":1430424776594, - "elapsedTime":7199 + "elapsedTime":7199}, + { + "appId":"application_1430424020775_0001", + "currentAppAttemptId":"appattempt_1430424020775_0001_000001", + "user":"zshen", + "name":"QuasiMonteCarlo", + "queue":"default", + "type":"MAPREDUCE", + "host":"localhost", + "rpcPort":56264, + "appState":"FINISHED", + "progress":100.0, + "diagnosticsInfo":"", + "originalTrackingUrl":"http://d-69-91-129-173.dhcp4.washington.edu:19888/jobhistory/job/job_1430424020775_0001", + "trackingUrl":"http://d-69-91-129-173.dhcp4.washington.edu:8088/proxy/application_1430424020775_0001/", + "finalAppStatus":"SUCCEEDED", + "submittedTime":1430424053809, + "startedTime":1430424072153, + "finishedTime":1430424776594, + "elapsedTime":18344, + "applicationTags":"mrapplication,ta-example" } ] } @@ -1227,6 +1247,7 @@ Response Body: 1430424053809 1430424072153 18344 + mrapplication,ta-example @@ -1275,7 +1296,8 @@ None | `allocatedVCores` | int | The sum of virtual cores allocated to the application's running containers | | `currentAppAttemptId` | string | The latest application attempt ID | | `host` | string | The host of the ApplicationMaster | -| `rpcPort` | int | The RPC port of the ApplicationMaster; zero if no IPC service declared. | +| `rpcPort` | int | The RPC port of the ApplicationMaster; zero if no IPC service declared | +| `applicationTags` | string | The application tags. | ### Response Examples: @@ -1311,7 +1333,8 @@ Response Body: "submittedTime": 1430424053809, "startedTime": 1430424053809, "finishedTime": 1430424072153, - "elapsedTime": 18344 + "elapsedTime": 18344, + "applicationTags": mrapplication,tag-example } #### XML response @@ -1349,6 +1372,7 @@ Response Body: 1430424053809 1430424072153 18344 + mrapplication,ta-example ## Application Attempt List