From 8d0ef3163287d811381e94cc0101dbc9847ee2db Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Mon, 8 Jun 2015 23:14:58 -0700 Subject: [PATCH] YARN-3787. Allowed generic history service to load a number of applications whose started time is within the given range. Contributed by Xuan Gong. --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../hadoop/yarn/webapp/YarnWebParams.java | 2 ++ .../ApplicationHistoryClientService.java | 9 +++++- .../ApplicationHistoryManager.java | 12 +++++--- .../ApplicationHistoryManagerImpl.java | 4 +-- ...licationHistoryManagerOnTimelineStore.java | 15 ++++++---- ...licationHistoryManagerOnTimelineStore.java | 10 ++++++- .../hadoop/yarn/server/webapp/AppsBlock.java | 30 +++++++++++++++++++ .../yarn/server/webapp/WebServices.java | 9 ++---- 9 files changed, 73 insertions(+), 21 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 87b3da3abd..cd00cf47a6 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -300,6 +300,9 @@ Release 2.8.0 - UNRELEASED YARN-3786. Document yarn class path options. (Brahma Reddy Battula via cnauroth) + YARN-3787. Allowed generic history service to load a number of applications whose + started time is within the given range. (Xuan Gong via zjshen) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java index 199f4cc17c..679e1d6d5d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java @@ -33,6 +33,8 @@ public interface YarnWebParams { String ENTITY_STRING = "entity.string"; String APP_OWNER = "app.owner"; String APP_STATE = "app.state"; + String APP_START_TIME_BEGIN = "app.started-time.begin"; + String APP_START_TIME_END = "app.started-time.end"; String APPS_NUM = "apps.num"; String QUEUE_NAME = "queue.name"; String NODE_STATE = "node.state"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java index 74279266c6..73d5d39277 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java @@ -184,9 +184,16 @@ public GetApplicationReportResponse getApplicationReport( public GetApplicationsResponse getApplications(GetApplicationsRequest request) throws YarnException, IOException { + long startedBegin = + request.getStartRange() == null ? 0L : request.getStartRange() + .getMinimumLong(); + long startedEnd = + request.getStartRange() == null ? Long.MAX_VALUE : request + .getStartRange().getMaximumLong(); GetApplicationsResponse response = GetApplicationsResponse.newInstance(new ArrayList( - history.getApplications(request.getLimit()).values())); + history.getApplications(request.getLimit(), startedBegin, startedEnd) + .values())); return response; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java index c218eec000..356b36781f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java @@ -51,10 +51,14 @@ ApplicationReport getApplication(ApplicationId appId) throws YarnException, IOException; /** - * This method returns the given number of Application + * This method returns the given number of Application in the + * given appStartedTime period. + * * {@link ApplicationReport}s. * * @param appsNum + * @param appStartedTimeBegin + * @param appStartedTimeEnd * * @return map of {@link ApplicationId} to {@link ApplicationReport}s. * @throws YarnException @@ -62,9 +66,9 @@ ApplicationReport getApplication(ApplicationId appId) throws YarnException, */ @Public @Unstable - Map - getApplications(long appsNum) throws YarnException, - IOException; + Map getApplications(long appsNum, + long appStartedTimeBegin, long appStartedTimeEnd) throws YarnException, + IOException; /** * Application can have multiple application attempts diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java index 479858f35b..130bb32b0e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java @@ -98,8 +98,8 @@ public ContainerReport getAMContainer(ApplicationAttemptId appAttemptId) } @Override - public Map getApplications(long appsNum) - throws IOException { + public Map getApplications(long appsNum, + long appStartedTimeBegin, long appStartedTimeEnd) throws IOException { Map histData = historyStore.getAllApplications(); HashMap applicationsReport = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index 0c7fdc0a9b..c7b19d360f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -109,12 +109,15 @@ public ApplicationReport getApplication(ApplicationId appId) } @Override - public Map getApplications(long appsNum) - throws YarnException, IOException { - TimelineEntities entities = timelineDataManager.getEntities( - ApplicationMetricsConstants.ENTITY_TYPE, null, null, null, null, null, - null, appsNum == Long.MAX_VALUE ? this.maxLoadedApplications : appsNum, - EnumSet.allOf(Field.class), UserGroupInformation.getLoginUser()); + public Map getApplications(long appsNum, + long appStartedTimeBegin, long appStartedTimeEnd) throws YarnException, + IOException { + TimelineEntities entities = + timelineDataManager.getEntities( + ApplicationMetricsConstants.ENTITY_TYPE, null, null, + appStartedTimeBegin, appStartedTimeEnd, null, null, + appsNum == Long.MAX_VALUE ? this.maxLoadedApplications : appsNum, + EnumSet.allOf(Field.class), UserGroupInformation.getLoginUser()); Map apps = new LinkedHashMap(); if (entities != null && entities.getEntities() != null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java index dbd75acfea..1e5dc5de8a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java @@ -342,13 +342,21 @@ public ContainerReport run() throws Exception { @Test public void testGetApplications() throws Exception { Collection apps = - historyManager.getApplications(Long.MAX_VALUE).values(); + historyManager.getApplications(Long.MAX_VALUE, 0L, Long.MAX_VALUE) + .values(); Assert.assertNotNull(apps); Assert.assertEquals(SCALE + 1, apps.size()); ApplicationId ignoredAppId = ApplicationId.newInstance(0, SCALE + 2); for (ApplicationReport app : apps) { Assert.assertNotEquals(ignoredAppId, app.getApplicationId()); } + + // Get apps by given appStartedTime period + apps = + historyManager.getApplications(Long.MAX_VALUE, 2147483653L, + Long.MAX_VALUE).values(); + Assert.assertNotNull(apps); + Assert.assertEquals(2, apps.size()); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java index b0a3f2bcf6..19ab1ce50f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java @@ -20,6 +20,8 @@ import static org.apache.hadoop.yarn.util.StringHelper.join; import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_START_TIME_BEGIN; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_START_TIME_END; import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPS_NUM; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE; @@ -30,6 +32,7 @@ import java.util.EnumSet; import org.apache.commons.lang.StringEscapeUtils; +import org.apache.commons.lang.math.LongRange; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.security.UserGroupInformation; @@ -40,6 +43,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.webapp.dao.AppInfo; +import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY; @@ -79,6 +83,32 @@ protected void fetchData() throws YarnException, IOException, long appsNum = Long.parseLong(appsNumStr); request.setLimit(appsNum); } + + String appStartedTimeBegainStr = $(APP_START_TIME_BEGIN); + long appStartedTimeBegain = 0; + if (appStartedTimeBegainStr != null && !appStartedTimeBegainStr.isEmpty()) { + appStartedTimeBegain = Long.parseLong(appStartedTimeBegainStr); + if (appStartedTimeBegain < 0) { + throw new BadRequestException( + "app.started-time.begin must be greater than 0"); + } + } + String appStartedTimeEndStr = $(APP_START_TIME_END); + long appStartedTimeEnd = Long.MAX_VALUE; + if (appStartedTimeEndStr != null && !appStartedTimeEndStr.isEmpty()) { + appStartedTimeEnd = Long.parseLong(appStartedTimeEndStr); + if (appStartedTimeEnd < 0) { + throw new BadRequestException( + "app.started-time.end must be greater than 0"); + } + } + if (appStartedTimeBegain > appStartedTimeEnd) { + throw new BadRequestException( + "app.started-time.end must be greater than app.started-time.begin"); + } + request.setStartRange( + new LongRange(appStartedTimeBegain, appStartedTimeEnd)); + if (callerUGI == null) { appReports = appBaseProt.getApplications(request).getApplicationList(); } else { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java index 3064145627..8c282637e8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java @@ -29,6 +29,7 @@ import javax.servlet.http.HttpServletResponse; import javax.ws.rs.WebApplicationException; +import org.apache.commons.lang.math.LongRange; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AuthorizationException; import org.apache.hadoop.util.StringUtils; @@ -75,7 +76,6 @@ public AppsInfo getApps(HttpServletRequest req, HttpServletResponse res, String startedEnd, String finishBegin, String finishEnd, Set applicationTypes) { UserGroupInformation callerUGI = getUser(req); - boolean checkStart = false; boolean checkEnd = false; boolean checkAppTypes = false; boolean checkAppStates = false; @@ -95,14 +95,12 @@ public AppsInfo getApps(HttpServletRequest req, HttpServletResponse res, } if (startedBegin != null && !startedBegin.isEmpty()) { - checkStart = true; sBegin = Long.parseLong(startedBegin); if (sBegin < 0) { throw new BadRequestException("startedTimeBegin must be greater than 0"); } } if (startedEnd != null && !startedEnd.isEmpty()) { - checkStart = true; sEnd = Long.parseLong(startedEnd); if (sEnd < 0) { throw new BadRequestException("startedTimeEnd must be greater than 0"); @@ -151,6 +149,7 @@ public AppsInfo getApps(HttpServletRequest req, HttpServletResponse res, final GetApplicationsRequest request = GetApplicationsRequest.newInstance(); request.setLimit(countNum); + request.setStartRange(new LongRange(sBegin, sEnd)); try { if (callerUGI == null) { // TODO: the request should take the params like what RMWebServices does @@ -198,10 +197,6 @@ public Collection run() throws Exception { continue; } - if (checkStart - && (appReport.getStartTime() < sBegin || appReport.getStartTime() > sEnd)) { - continue; - } if (checkEnd && (appReport.getFinishTime() < fBegin || appReport.getFinishTime() > fEnd)) { continue;