Revert "YARN-1462. Made RM write application tags to timeline server and exposed them to users via generic history web UI and REST API. Contributed by Xuan Gong."

This reverts commit 4a9ec1a824.
This commit is contained in:
Zhijie Shen 2015-06-03 14:15:41 -07:00
parent 4eec2fd132
commit bc85959edd
20 changed files with 36 additions and 144 deletions

View File

@ -90,7 +90,7 @@ private ApplicationReport getUnknownApplicationReport() {
return ApplicationReport.newInstance(unknownAppId, unknownAttemptId, return ApplicationReport.newInstance(unknownAppId, unknownAttemptId,
"N/A", "N/A", "N/A", "N/A", 0, null, YarnApplicationState.NEW, "N/A", "N/A", "N/A", "N/A", "N/A", 0, null, YarnApplicationState.NEW, "N/A",
"N/A", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f, "N/A", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f,
YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, null); YarnConfiguration.DEFAULT_APPLICATION_TYPE, null);
} }
NotRunningJob(ApplicationReport applicationReport, JobState jobState) { NotRunningJob(ApplicationReport applicationReport, JobState jobState) {

View File

@ -516,7 +516,7 @@ private ApplicationReport getFinishedApplicationReport() {
return ApplicationReport.newInstance(appId, attemptId, "user", "queue", return ApplicationReport.newInstance(appId, attemptId, "user", "queue",
"appname", "host", 124, null, YarnApplicationState.FINISHED, "appname", "host", 124, null, YarnApplicationState.FINISHED,
"diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null, "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null,
"N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, null); "N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null);
} }
private ApplicationReport getRunningApplicationReport(String host, int port) { private ApplicationReport getRunningApplicationReport(String host, int port) {
@ -526,7 +526,7 @@ private ApplicationReport getRunningApplicationReport(String host, int port) {
return ApplicationReport.newInstance(appId, attemptId, "user", "queue", return ApplicationReport.newInstance(appId, attemptId, "user", "queue",
"appname", host, port, null, YarnApplicationState.RUNNING, "diagnostics", "appname", host, port, null, YarnApplicationState.RUNNING, "diagnostics",
"url", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f, "url", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f,
YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, null); YarnConfiguration.DEFAULT_APPLICATION_TYPE, null);
} }
private ResourceMgrDelegate getRMDelegate() throws IOException { private ResourceMgrDelegate getRMDelegate() throws IOException {

View File

@ -196,7 +196,7 @@ public ClientServiceDelegate answer(InvocationOnMock invocation)
ApplicationReport.newInstance(appId, null, "tmp", "tmp", "tmp", ApplicationReport.newInstance(appId, null, "tmp", "tmp", "tmp",
"tmp", 0, null, YarnApplicationState.FINISHED, "tmp", "tmp", "tmp", 0, null, YarnApplicationState.FINISHED, "tmp", "tmp",
0l, 0l, FinalApplicationStatus.SUCCEEDED, null, null, 0f, 0l, 0l, FinalApplicationStatus.SUCCEEDED, null, null, 0f,
"tmp", null, null)); "tmp", null));
yarnRunner.killJob(jobId); yarnRunner.killJob(jobId);
verify(clientDelegate).killJob(jobId); verify(clientDelegate).killJob(jobId);
} }

View File

@ -501,9 +501,6 @@ Release 2.7.1 - UNRELEASED
YARN-3723. Need to clearly document primaryFilter and otherInfo value type. YARN-3723. Need to clearly document primaryFilter and otherInfo value type.
(Zhijie Shen via xgong) (Zhijie Shen via xgong)
YARN-1462. Made RM write application tags to timeline server and exposed them
to users via generic history web UI and REST API. (Xuan Gong via zjshen)
OPTIMIZATIONS OPTIMIZATIONS
BUG FIXES BUG FIXES

View File

@ -59,8 +59,7 @@ public static ApplicationReport newInstance(ApplicationId applicationId,
YarnApplicationState state, String diagnostics, String url, YarnApplicationState state, String diagnostics, String url,
long startTime, long finishTime, FinalApplicationStatus finalStatus, long startTime, long finishTime, FinalApplicationStatus finalStatus,
ApplicationResourceUsageReport appResources, String origTrackingUrl, ApplicationResourceUsageReport appResources, String origTrackingUrl,
float progress, String applicationType, Token amRmToken, float progress, String applicationType, Token amRmToken) {
Set<String> tags) {
ApplicationReport report = Records.newRecord(ApplicationReport.class); ApplicationReport report = Records.newRecord(ApplicationReport.class);
report.setApplicationId(applicationId); report.setApplicationId(applicationId);
report.setCurrentApplicationAttemptId(applicationAttemptId); report.setCurrentApplicationAttemptId(applicationAttemptId);
@ -81,7 +80,6 @@ public static ApplicationReport newInstance(ApplicationId applicationId,
report.setProgress(progress); report.setProgress(progress);
report.setApplicationType(applicationType); report.setApplicationType(applicationType);
report.setAMRMToken(amRmToken); report.setAMRMToken(amRmToken);
report.setApplicationTags(tags);
return report; return report;
} }

View File

@ -647,7 +647,7 @@ public ApplicationReport createFakeAppReport() {
"fakeQueue", "fakeApplicationName", "localhost", 0, null, "fakeQueue", "fakeApplicationName", "localhost", 0, null,
YarnApplicationState.FINISHED, "fake an application report", "", YarnApplicationState.FINISHED, "fake an application report", "",
1000l, 1200l, FinalApplicationStatus.FAILED, null, "", 50f, 1000l, 1200l, FinalApplicationStatus.FAILED, null, "", 50f,
"fakeApplicationType", null, null); "fakeApplicationType", null);
return report; return report;
} }

View File

@ -334,7 +334,7 @@ private void createAppReports() {
"queue", "appname", "host", 124, null, "queue", "appname", "host", 124, null,
YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN",
null, null); null);
List<ApplicationReport> applicationReports = List<ApplicationReport> applicationReports =
new ArrayList<ApplicationReport>(); new ArrayList<ApplicationReport>();
applicationReports.add(newApplicationReport); applicationReports.add(newApplicationReport);
@ -391,7 +391,7 @@ private void createAppReports() {
"queue2", "appname2", "host2", 125, null, "queue2", "appname2", "host2", 125, null,
YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f,
"NON-YARN", null, null); "NON-YARN", null);
applicationReports.add(newApplicationReport2); applicationReports.add(newApplicationReport2);
ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7); ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7);
@ -401,7 +401,7 @@ private void createAppReports() {
"queue3", "appname3", "host3", 126, null, "queue3", "appname3", "host3", 126, null,
YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f,
"MAPREDUCE", null, null); "MAPREDUCE", null);
applicationReports.add(newApplicationReport3); applicationReports.add(newApplicationReport3);
ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8); ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8);
@ -411,7 +411,7 @@ private void createAppReports() {
"queue4", "appname4", "host4", 127, null, "queue4", "appname4", "host4", 127, null,
YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f,
"NON-MAPREDUCE", null, null); "NON-MAPREDUCE", null);
applicationReports.add(newApplicationReport4); applicationReports.add(newApplicationReport4);
reports = applicationReports; reports = applicationReports;
} }

View File

@ -613,8 +613,7 @@ private List<ApplicationReport> createAppReports() {
applicationId, ApplicationAttemptId.newInstance(applicationId, 1), applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null, "user", "queue", "appname", "host", 124, null,
YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
null);
List<ApplicationReport> applicationReports = new ArrayList<ApplicationReport>(); List<ApplicationReport> applicationReports = new ArrayList<ApplicationReport>();
applicationReports.add(newApplicationReport); applicationReports.add(newApplicationReport);
List<ApplicationAttemptReport> appAttempts = new ArrayList<ApplicationAttemptReport>(); List<ApplicationAttemptReport> appAttempts = new ArrayList<ApplicationAttemptReport>();
@ -690,7 +689,7 @@ private List<ApplicationReport> createAppReports() {
"user2", "queue2", "appname2", "host2", 125, null, "user2", "queue2", "appname2", "host2", 125, null,
YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN", FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN",
null, null); null);
applicationReports.add(newApplicationReport2); applicationReports.add(newApplicationReport2);
ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7); ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7);
@ -699,7 +698,7 @@ private List<ApplicationReport> createAppReports() {
"user3", "queue3", "appname3", "host3", 126, null, "user3", "queue3", "appname3", "host3", 126, null,
YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE",
null, null); null);
applicationReports.add(newApplicationReport3); applicationReports.add(newApplicationReport3);
ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8); ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8);
@ -710,7 +709,7 @@ private List<ApplicationReport> createAppReports() {
"user4", "queue4", "appname4", "host4", 127, null, "user4", "queue4", "appname4", "host4", 127, null,
YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f,
"NON-MAPREDUCE", null, null); "NON-MAPREDUCE", null);
applicationReports.add(newApplicationReport4); applicationReports.add(newApplicationReport4);
return applicationReports; return applicationReports;
} }

View File

@ -104,7 +104,7 @@ public void testGetApplicationReport() throws Exception {
"user", "queue", "appname", "host", 124, null, "user", "queue", "appname", "host", 124, null,
YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN", FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN",
null, null); null);
newApplicationReport.setLogAggregationStatus(LogAggregationStatus.SUCCEEDED); newApplicationReport.setLogAggregationStatus(LogAggregationStatus.SUCCEEDED);
when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
newApplicationReport); newApplicationReport);
@ -350,8 +350,7 @@ public void testGetApplications() throws Exception {
applicationId, ApplicationAttemptId.newInstance(applicationId, 1), applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null, "user", "queue", "appname", "host", 124, null,
YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
null);
List<ApplicationReport> applicationReports = new ArrayList<ApplicationReport>(); List<ApplicationReport> applicationReports = new ArrayList<ApplicationReport>();
applicationReports.add(newApplicationReport); applicationReports.add(newApplicationReport);
@ -361,7 +360,7 @@ public void testGetApplications() throws Exception {
"user2", "queue2", "appname2", "host2", 125, null, "user2", "queue2", "appname2", "host2", 125, null,
YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2, YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN", FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN",
null, null); null);
applicationReports.add(newApplicationReport2); applicationReports.add(newApplicationReport2);
ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7); ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7);
@ -370,7 +369,7 @@ public void testGetApplications() throws Exception {
"user3", "queue3", "appname3", "host3", 126, null, "user3", "queue3", "appname3", "host3", 126, null,
YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3, YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE",
null, null); null);
applicationReports.add(newApplicationReport3); applicationReports.add(newApplicationReport3);
ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8); ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8);
@ -379,7 +378,7 @@ public void testGetApplications() throws Exception {
"user4", "queue4", "appname4", "host4", 127, null, "user4", "queue4", "appname4", "host4", 127, null,
YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4, YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE", FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE",
null, null); null);
applicationReports.add(newApplicationReport4); applicationReports.add(newApplicationReport4);
ApplicationId applicationId5 = ApplicationId.newInstance(1234, 9); ApplicationId applicationId5 = ApplicationId.newInstance(1234, 9);
@ -388,7 +387,7 @@ public void testGetApplications() throws Exception {
"user5", "queue5", "appname5", "host5", 128, null, "user5", "queue5", "appname5", "host5", 128, null,
YarnApplicationState.ACCEPTED, "diagnostics5", "url5", 5, 5, YarnApplicationState.ACCEPTED, "diagnostics5", "url5", 5, 5,
FinalApplicationStatus.KILLED, null, "N/A", 0.93789f, "HIVE", FinalApplicationStatus.KILLED, null, "N/A", 0.93789f, "HIVE",
null, null); null);
applicationReports.add(newApplicationReport5); applicationReports.add(newApplicationReport5);
ApplicationId applicationId6 = ApplicationId.newInstance(1234, 10); ApplicationId applicationId6 = ApplicationId.newInstance(1234, 10);
@ -397,7 +396,7 @@ public void testGetApplications() throws Exception {
"user6", "queue6", "appname6", "host6", 129, null, "user6", "queue6", "appname6", "host6", 129, null,
YarnApplicationState.SUBMITTED, "diagnostics6", "url6", 6, 6, YarnApplicationState.SUBMITTED, "diagnostics6", "url6", 6, 6,
FinalApplicationStatus.KILLED, null, "N/A", 0.99789f, "PIG", FinalApplicationStatus.KILLED, null, "N/A", 0.99789f, "PIG",
null, null); null);
applicationReports.add(newApplicationReport6); applicationReports.add(newApplicationReport6);
// Test command yarn application -list // Test command yarn application -list
@ -821,8 +820,7 @@ public void testKillApplication() throws Exception {
applicationId, ApplicationAttemptId.newInstance(applicationId, 1), applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null, "user", "queue", "appname", "host", 124, null,
YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
null);
when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
newApplicationReport2); newApplicationReport2);
int result = cli.run(new String[] { "application","-kill", applicationId.toString() }); int result = cli.run(new String[] { "application","-kill", applicationId.toString() });
@ -835,8 +833,7 @@ public void testKillApplication() throws Exception {
applicationId, ApplicationAttemptId.newInstance(applicationId, 1), applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null, "user", "queue", "appname", "host", 124, null,
YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
null);
when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
newApplicationReport); newApplicationReport);
result = cli.run(new String[] { "application","-kill", applicationId.toString() }); result = cli.run(new String[] { "application","-kill", applicationId.toString() });
@ -871,8 +868,7 @@ public void testMoveApplicationAcrossQueues() throws Exception {
applicationId, ApplicationAttemptId.newInstance(applicationId, 1), applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null, "user", "queue", "appname", "host", 124, null,
YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0, YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
null);
when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
newApplicationReport2); newApplicationReport2);
int result = cli.run(new String[] { "application", "-movetoqueue", int result = cli.run(new String[] { "application", "-movetoqueue",
@ -887,8 +883,7 @@ public void testMoveApplicationAcrossQueues() throws Exception {
applicationId, ApplicationAttemptId.newInstance(applicationId, 1), applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null, "user", "queue", "appname", "host", 124, null,
YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0, YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null, FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
null);
when(client.getApplicationReport(any(ApplicationId.class))).thenReturn( when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
newApplicationReport); newApplicationReport);
result = cli.run(new String[] { "application", "-movetoqueue", result = cli.run(new String[] { "application", "-movetoqueue",

View File

@ -58,8 +58,7 @@ protected static ApplicationReport createApplicationReport(
ApplicationReport.newInstance(appId, appAttemptId, "user", "queue", ApplicationReport.newInstance(appId, appAttemptId, "user", "queue",
"appname", "host", 124, null, YarnApplicationState.FINISHED, "appname", "host", 124, null, YarnApplicationState.FINISHED,
"diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null, "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null,
"N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, "N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null);
null);
return appReport; return appReport;
} }

View File

@ -139,7 +139,7 @@ private ApplicationReport convertToApplicationReport(
appHistory.getYarnApplicationState(), appHistory.getDiagnosticsInfo(), appHistory.getYarnApplicationState(), appHistory.getDiagnosticsInfo(),
trackingUrl, appHistory.getStartTime(), appHistory.getFinishTime(), trackingUrl, appHistory.getStartTime(), appHistory.getFinishTime(),
appHistory.getFinalApplicationStatus(), null, "", 100, appHistory.getFinalApplicationStatus(), null, "", 100,
appHistory.getApplicationType(), null, null); appHistory.getApplicationType(), null);
} }
private ApplicationAttemptHistoryData getLastAttempt(ApplicationId appId) private ApplicationAttemptHistoryData getLastAttempt(ApplicationId appId)

View File

@ -19,14 +19,11 @@
package org.apache.hadoop.yarn.server.applicationhistoryservice; package org.apache.hadoop.yarn.server.applicationhistoryservice;
import java.io.IOException; import java.io.IOException;
import java.util.Collection;
import java.util.EnumSet; import java.util.EnumSet;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
@ -252,7 +249,6 @@ private static ApplicationReportExt convertToApplicationReport(
FinalApplicationStatus finalStatus = FinalApplicationStatus.UNDEFINED; FinalApplicationStatus finalStatus = FinalApplicationStatus.UNDEFINED;
YarnApplicationState state = YarnApplicationState.ACCEPTED; YarnApplicationState state = YarnApplicationState.ACCEPTED;
ApplicationResourceUsageReport appResources = null; ApplicationResourceUsageReport appResources = null;
Set<String> appTags = null;
Map<ApplicationAccessType, String> appViewACLs = Map<ApplicationAccessType, String> appViewACLs =
new HashMap<ApplicationAccessType, String>(); new HashMap<ApplicationAccessType, String>();
Map<String, Object> entityInfo = entity.getOtherInfo(); Map<String, Object> entityInfo = entity.getOtherInfo();
@ -274,7 +270,7 @@ private static ApplicationReportExt convertToApplicationReport(
ConverterUtils.toApplicationId(entity.getEntityId()), ConverterUtils.toApplicationId(entity.getEntityId()),
latestApplicationAttemptId, user, queue, name, null, -1, null, state, latestApplicationAttemptId, user, queue, name, null, -1, null, state,
diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null, diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null,
null, progress, type, null, appTags), appViewACLs); null, progress, type, null), appViewACLs);
} }
if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) { if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) {
queue = queue =
@ -299,17 +295,6 @@ private static ApplicationReportExt convertToApplicationReport(
appResources=ApplicationResourceUsageReport appResources=ApplicationResourceUsageReport
.newInstance(0, 0, null, null, null, memorySeconds, vcoreSeconds); .newInstance(0, 0, null, null, null, memorySeconds, vcoreSeconds);
} }
if (entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) {
appTags = new HashSet<String>();
Object obj = entityInfo.get(ApplicationMetricsConstants.APP_TAGS_INFO);
if (obj != null && obj instanceof Collection<?>) {
for(Object o : (Collection<?>)obj) {
if (o != null) {
appTags.add(o.toString());
}
}
}
}
} }
List<TimelineEvent> events = entity.getEvents(); List<TimelineEvent> events = entity.getEvents();
if (events != null) { if (events != null) {
@ -362,7 +347,7 @@ private static ApplicationReportExt convertToApplicationReport(
ConverterUtils.toApplicationId(entity.getEntityId()), ConverterUtils.toApplicationId(entity.getEntityId()),
latestApplicationAttemptId, user, queue, name, null, -1, null, state, latestApplicationAttemptId, user, queue, name, null, -1, null, state,
diagnosticsInfo, null, createdTime, finishedTime, finalStatus, appResources, diagnosticsInfo, null, createdTime, finishedTime, finalStatus, appResources,
null, progress, type, null, appTags), appViewACLs); null, progress, type, null), appViewACLs);
} }
private static ApplicationAttemptReport convertToApplicationAttemptReport( private static ApplicationAttemptReport convertToApplicationAttemptReport(

View File

@ -22,9 +22,7 @@
import java.util.Arrays; import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet;
import java.util.Map; import java.util.Map;
import java.util.Set;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.SaslRpcServer.AuthMethod; import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
@ -187,9 +185,6 @@ public ApplicationReport run() throws Exception {
Assert.assertEquals(Integer.MAX_VALUE + 3L Assert.assertEquals(Integer.MAX_VALUE + 3L
+ +app.getApplicationId().getId(), app.getFinishTime()); + +app.getApplicationId().getId(), app.getFinishTime());
Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001); Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001);
Assert.assertEquals(2, app.getApplicationTags().size());
Assert.assertTrue(app.getApplicationTags().contains("Test_APP_TAGS_1"));
Assert.assertTrue(app.getApplicationTags().contains("Test_APP_TAGS_2"));
// App 2 doesn't have the ACLs, such that the default ACLs " " will be used. // App 2 doesn't have the ACLs, such that the default ACLs " " will be used.
// Nobody except admin and owner has access to the details of the app. // Nobody except admin and owner has access to the details of the app.
if ((i == 1 && callerUGI != null && if ((i == 1 && callerUGI != null &&
@ -476,10 +471,6 @@ private static TimelineEntity createApplicationTimelineEntity(
entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO,
"user2"); "user2");
} }
Set<String> appTags = new HashSet<String>();
appTags.add("Test_APP_TAGS_1");
appTags.add("Test_APP_TAGS_2");
entityInfo.put(ApplicationMetricsConstants.APP_TAGS_INFO, appTags);
entity.setOtherInfo(entityInfo); entity.setOtherInfo(entityInfo);
TimelineEvent tEvent = new TimelineEvent(); TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType(ApplicationMetricsConstants.CREATED_EVENT_TYPE); tEvent.setEventType(ApplicationMetricsConstants.CREATED_EVENT_TYPE);

View File

@ -73,5 +73,4 @@ public class ApplicationMetricsConstants {
public static final String LATEST_APP_ATTEMPT_EVENT_INFO = public static final String LATEST_APP_ATTEMPT_EVENT_INFO =
"YARN_APPLICATION_LATEST_APP_ATTEMPT"; "YARN_APPLICATION_LATEST_APP_ATTEMPT";
public static final String APP_TAGS_INFO = "YARN_APPLICATION_TAGS";
} }

View File

@ -18,8 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.metrics; package org.apache.hadoop.yarn.server.resourcemanager.metrics;
import java.util.Set;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
public class ApplicationCreatedEvent extends public class ApplicationCreatedEvent extends
@ -31,7 +29,6 @@ public class ApplicationCreatedEvent extends
private String user; private String user;
private String queue; private String queue;
private long submittedTime; private long submittedTime;
private Set<String> appTags;
public ApplicationCreatedEvent(ApplicationId appId, public ApplicationCreatedEvent(ApplicationId appId,
String name, String name,
@ -39,8 +36,7 @@ public ApplicationCreatedEvent(ApplicationId appId,
String user, String user,
String queue, String queue,
long submittedTime, long submittedTime,
long createdTime, long createdTime) {
Set<String> appTags) {
super(SystemMetricsEventType.APP_CREATED, createdTime); super(SystemMetricsEventType.APP_CREATED, createdTime);
this.appId = appId; this.appId = appId;
this.name = name; this.name = name;
@ -48,7 +44,6 @@ public ApplicationCreatedEvent(ApplicationId appId,
this.user = user; this.user = user;
this.queue = queue; this.queue = queue;
this.submittedTime = submittedTime; this.submittedTime = submittedTime;
this.appTags = appTags;
} }
@Override @Override
@ -80,7 +75,4 @@ public long getSubmittedTime() {
return submittedTime; return submittedTime;
} }
public Set<String> getAppTags() {
return appTags;
}
} }

View File

@ -106,7 +106,7 @@ public void appCreated(RMApp app, long createdTime) {
app.getUser(), app.getUser(),
app.getQueue(), app.getQueue(),
app.getSubmitTime(), app.getSubmitTime(),
createdTime, app.getApplicationTags())); createdTime));
} }
} }
@ -251,8 +251,6 @@ private void publishApplicationCreatedEvent(ApplicationCreatedEvent event) {
event.getQueue()); event.getQueue());
entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO, entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO,
event.getSubmittedTime()); event.getSubmittedTime());
entityInfo.put(ApplicationMetricsConstants.APP_TAGS_INFO,
event.getAppTags());
entity.setOtherInfo(entityInfo); entity.setOtherInfo(entityInfo);
TimelineEvent tEvent = new TimelineEvent(); TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType( tEvent.setEventType(

View File

@ -301,7 +301,7 @@ public ApplicationReport createAndGetApplicationReport(
getName(), null, 0, null, null, getDiagnostics().toString(), getName(), null, 0, null, null, getDiagnostics().toString(),
getTrackingUrl(), getStartTime(), getFinishTime(), getTrackingUrl(), getStartTime(), getFinishTime(),
getFinalApplicationStatus(), usageReport , null, getProgress(), getFinalApplicationStatus(), usageReport , null, getProgress(),
type, null, null); type, null);
return report; return report;
} }

View File

@ -21,11 +21,7 @@
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
import java.util.Collection;
import java.util.EnumSet; import java.util.EnumSet;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
@ -147,8 +143,6 @@ public void testPublishApplicationMetrics() throws Exception {
Assert.assertEquals(app.getSubmitTime(), Assert.assertEquals(app.getSubmitTime(),
entity.getOtherInfo().get( entity.getOtherInfo().get(
ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO)); ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO));
Assert.assertTrue(verifyAppTags(app.getApplicationTags(),
entity.getOtherInfo()));
if (i == 1) { if (i == 1) {
Assert.assertEquals("uers1,user2", Assert.assertEquals("uers1,user2",
entity.getOtherInfo().get( entity.getOtherInfo().get(
@ -358,10 +352,6 @@ private static RMApp createRMApp(ApplicationId appId) {
FinalApplicationStatus.UNDEFINED); FinalApplicationStatus.UNDEFINED);
when(app.getRMAppMetrics()).thenReturn( when(app.getRMAppMetrics()).thenReturn(
new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE)); new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE));
Set<String> appTags = new HashSet<String>();
appTags.add("test");
appTags.add("tags");
when(app.getApplicationTags()).thenReturn(appTags);
return app; return app;
} }
@ -402,31 +392,4 @@ private static RMContainer createRMContainer(ContainerId containerId) {
return container; return container;
} }
private static boolean verifyAppTags(Set<String> appTags,
Map<String, Object> entityInfo) {
if (!entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) {
return false;
}
Object obj = entityInfo.get(ApplicationMetricsConstants.APP_TAGS_INFO);
if (obj instanceof Collection<?>) {
Collection<?> collection = (Collection<?>) obj;
if (collection.size() != appTags.size()) {
return false;
}
for (String appTag : appTags) {
boolean match = false;
for (Object o : collection) {
if (o.toString().equals(appTag)) {
match = true;
break;
}
}
if (!match) {
return false;
}
}
return true;
}
return false;
}
} }

View File

@ -253,7 +253,7 @@ public static ClientRMService mockClientRMService(RMContext rmContext) {
app.getStartTime(), app.getFinishTime(), app.getStartTime(), app.getFinishTime(),
app.getFinalApplicationStatus(), app.getFinalApplicationStatus(),
(ApplicationResourceUsageReport) null, app.getTrackingUrl(), (ApplicationResourceUsageReport) null, app.getTrackingUrl(),
app.getProgress(), app.getApplicationType(), (Token) null, null); app.getProgress(), app.getApplicationType(), (Token) null);
appReports.add(appReport); appReports.add(appReport);
} }
GetApplicationsResponse response = mock(GetApplicationsResponse.class); GetApplicationsResponse response = mock(GetApplicationsResponse.class);

View File

@ -1125,27 +1125,7 @@ Response Body:
"submittedTime":1430424769395, "submittedTime":1430424769395,
"startedTime":1430424769395, "startedTime":1430424769395,
"finishedTime":1430424776594, "finishedTime":1430424776594,
"elapsedTime":7199}, "elapsedTime":7199
{
"appId":"application_1430424020775_0001",
"currentAppAttemptId":"appattempt_1430424020775_0001_000001",
"user":"zshen",
"name":"QuasiMonteCarlo",
"queue":"default",
"type":"MAPREDUCE",
"host":"localhost",
"rpcPort":56264,
"appState":"FINISHED",
"progress":100.0,
"diagnosticsInfo":"",
"originalTrackingUrl":"http://d-69-91-129-173.dhcp4.washington.edu:19888/jobhistory/job/job_1430424020775_0001",
"trackingUrl":"http://d-69-91-129-173.dhcp4.washington.edu:8088/proxy/application_1430424020775_0001/",
"finalAppStatus":"SUCCEEDED",
"submittedTime":1430424053809,
"startedTime":1430424072153,
"finishedTime":1430424776594,
"elapsedTime":18344,
"applicationTags":"mrapplication,ta-example"
} }
] ]
} }
@ -1247,7 +1227,6 @@ Response Body:
<startedTime>1430424053809</startedTime> <startedTime>1430424053809</startedTime>
<finishedTime>1430424072153</finishedTime> <finishedTime>1430424072153</finishedTime>
<elapsedTime>18344</elapsedTime> <elapsedTime>18344</elapsedTime>
<applicationTags>mrapplication,ta-example</applicationTags>
</app> </app>
</apps> </apps>
@ -1296,8 +1275,7 @@ None
| `allocatedVCores` | int | The sum of virtual cores allocated to the application's running containers | | `allocatedVCores` | int | The sum of virtual cores allocated to the application's running containers |
| `currentAppAttemptId` | string | The latest application attempt ID | | `currentAppAttemptId` | string | The latest application attempt ID |
| `host` | string | The host of the ApplicationMaster | | `host` | string | The host of the ApplicationMaster |
| `rpcPort` | int | The RPC port of the ApplicationMaster; zero if no IPC service declared | | `rpcPort` | int | The RPC port of the ApplicationMaster; zero if no IPC service declared. |
| `applicationTags` | string | The application tags. |
### Response Examples: ### Response Examples:
@ -1333,8 +1311,7 @@ Response Body:
"submittedTime": 1430424053809, "submittedTime": 1430424053809,
"startedTime": 1430424053809, "startedTime": 1430424053809,
"finishedTime": 1430424072153, "finishedTime": 1430424072153,
"elapsedTime": 18344, "elapsedTime": 18344
"applicationTags": mrapplication,tag-example
} }
#### XML response #### XML response
@ -1372,7 +1349,6 @@ Response Body:
<startedTime>1430424053809</startedTime> <startedTime>1430424053809</startedTime>
<finishedTime>1430424072153</finishedTime> <finishedTime>1430424072153</finishedTime>
<elapsedTime>18344</elapsedTime> <elapsedTime>18344</elapsedTime>
<applicationTags>mrapplication,ta-example</applicationTags>
</app> </app>
## <a name="REST_API_APPLICATION_ATTEMPT_LIST"></a>Application Attempt List ## <a name="REST_API_APPLICATION_ATTEMPT_LIST"></a>Application Attempt List