big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-3787. Allowed generic history service to load a number of applications whose started time is within the given range. Contributed by Xuan Gong.

Browse Source
This commit is contained in:
Zhijie Shen 2015-06-08 23:14:58 -07:00
parent c45784bc90
commit 8d0ef31632
9 changed files with 73 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -300,6 +300,9 @@ Release 2.8.0 - UNRELEASED
YARN-3786. Document yarn class path options.
(Brahma Reddy Battula via cnauroth)
YARN-3787. Allowed generic history service to load a number of applications whose
started time is within the given range. (Xuan Gong via zjshen)
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
View File

@ -33,6 +33,8 @@ public interface YarnWebParams {
String ENTITY_STRING = "entity.string";
String APP_OWNER = "app.owner";
String APP_STATE = "app.state";
String APP_START_TIME_BEGIN = "app.started-time.begin";
String APP_START_TIME_END = "app.started-time.end";
String APPS_NUM = "apps.num";
String QUEUE_NAME = "queue.name";
String NODE_STATE = "node.state";

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
View File

@ -184,9 +184,16 @@ public GetApplicationReportResponse getApplicationReport(
public GetApplicationsResponse
getApplications(GetApplicationsRequest request) throws YarnException,
IOException {
long startedBegin =
request.getStartRange() == null ? 0L : request.getStartRange()
.getMinimumLong();
long startedEnd =
request.getStartRange() == null ? Long.MAX_VALUE : request
.getStartRange().getMaximumLong();
GetApplicationsResponse response =
GetApplicationsResponse.newInstance(new ArrayList<ApplicationReport>(
history.getApplications(request.getLimit()).values()));
history.getApplications(request.getLimit(), startedBegin, startedEnd)
.values()));
return response;
}

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java
View File

@ -51,10 +51,14 @@ ApplicationReport getApplication(ApplicationId appId) throws YarnException,
IOException;
/**
* This method returns the given number of Application
* This method returns the given number of Application in the
* given appStartedTime period.
*
* {@link ApplicationReport}s.
*
* @param appsNum
* @param appStartedTimeBegin
* @param appStartedTimeEnd
*
* @return map of {@link ApplicationId} to {@link ApplicationReport}s.
* @throws YarnException
@ -62,8 +66,8 @@ ApplicationReport getApplication(ApplicationId appId) throws YarnException,
*/
@Public
@Unstable
Map<ApplicationId, ApplicationReport>
getApplications(long appsNum) throws YarnException,
Map<ApplicationId, ApplicationReport> getApplications(long appsNum,
long appStartedTimeBegin, long appStartedTimeEnd) throws YarnException,
IOException;
/**

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
View File

@ -98,8 +98,8 @@ public ContainerReport getAMContainer(ApplicationAttemptId appAttemptId)
}
@Override
public Map<ApplicationId, ApplicationReport> getApplications(long appsNum)
throws IOException {
public Map<ApplicationId, ApplicationReport> getApplications(long appsNum,
long appStartedTimeBegin, long appStartedTimeEnd) throws IOException {
Map<ApplicationId, ApplicationHistoryData> histData =
historyStore.getAllApplications();
HashMap<ApplicationId, ApplicationReport> applicationsReport =

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
View File

@ -109,11 +109,14 @@ public ApplicationReport getApplication(ApplicationId appId)
}
@Override
public Map<ApplicationId, ApplicationReport> getApplications(long appsNum)
throws YarnException, IOException {
TimelineEntities entities = timelineDataManager.getEntities(
ApplicationMetricsConstants.ENTITY_TYPE, null, null, null, null, null,
null, appsNum == Long.MAX_VALUE ? this.maxLoadedApplications : appsNum,
public Map<ApplicationId, ApplicationReport> getApplications(long appsNum,
long appStartedTimeBegin, long appStartedTimeEnd) throws YarnException,
IOException {
TimelineEntities entities =
timelineDataManager.getEntities(
ApplicationMetricsConstants.ENTITY_TYPE, null, null,
appStartedTimeBegin, appStartedTimeEnd, null, null,
appsNum == Long.MAX_VALUE ? this.maxLoadedApplications : appsNum,
EnumSet.allOf(Field.class), UserGroupInformation.getLoginUser());
Map<ApplicationId, ApplicationReport> apps =
new LinkedHashMap<ApplicationId, ApplicationReport>();

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
View File

@ -342,13 +342,21 @@ public ContainerReport run() throws Exception {
@Test
public void testGetApplications() throws Exception {
Collection<ApplicationReport> apps =
historyManager.getApplications(Long.MAX_VALUE).values();
historyManager.getApplications(Long.MAX_VALUE, 0L, Long.MAX_VALUE)
.values();
Assert.assertNotNull(apps);
Assert.assertEquals(SCALE + 1, apps.size());
ApplicationId ignoredAppId = ApplicationId.newInstance(0, SCALE + 2);
for (ApplicationReport app : apps) {
Assert.assertNotEquals(ignoredAppId, app.getApplicationId());
}
// Get apps by given appStartedTime period
apps =
historyManager.getApplications(Long.MAX_VALUE, 2147483653L,
Long.MAX_VALUE).values();
Assert.assertNotNull(apps);
Assert.assertEquals(2, apps.size());
}
@Test

30
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
View File

@ -20,6 +20,8 @@
import static org.apache.hadoop.yarn.util.StringHelper.join;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_START_TIME_BEGIN;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_START_TIME_END;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPS_NUM;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE;
@ -30,6 +32,7 @@
import java.util.EnumSet;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang.math.LongRange;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.security.UserGroupInformation;
@ -40,6 +43,7 @@
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
import org.apache.hadoop.yarn.webapp.BadRequestException;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
@ -79,6 +83,32 @@ protected void fetchData() throws YarnException, IOException,
long appsNum = Long.parseLong(appsNumStr);
request.setLimit(appsNum);
}
String appStartedTimeBegainStr = $(APP_START_TIME_BEGIN);
long appStartedTimeBegain = 0;
if (appStartedTimeBegainStr != null && !appStartedTimeBegainStr.isEmpty()) {
appStartedTimeBegain = Long.parseLong(appStartedTimeBegainStr);
if (appStartedTimeBegain < 0) {
throw new BadRequestException(
"app.started-time.begin must be greater than 0");
}
}
String appStartedTimeEndStr = $(APP_START_TIME_END);
long appStartedTimeEnd = Long.MAX_VALUE;
if (appStartedTimeEndStr != null && !appStartedTimeEndStr.isEmpty()) {
appStartedTimeEnd = Long.parseLong(appStartedTimeEndStr);
if (appStartedTimeEnd < 0) {
throw new BadRequestException(
"app.started-time.end must be greater than 0");
}
}
if (appStartedTimeBegain > appStartedTimeEnd) {
throw new BadRequestException(
"app.started-time.end must be greater than app.started-time.begin");
}
request.setStartRange(
new LongRange(appStartedTimeBegain, appStartedTimeEnd));
if (callerUGI == null) {
appReports = appBaseProt.getApplications(request).getApplicationList();
} else {

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
View File

@ -29,6 +29,7 @@
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.WebApplicationException;
import org.apache.commons.lang.math.LongRange;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AuthorizationException;
import org.apache.hadoop.util.StringUtils;
@ -75,7 +76,6 @@ public AppsInfo getApps(HttpServletRequest req, HttpServletResponse res,
String startedEnd, String finishBegin, String finishEnd,
Set<String> applicationTypes) {
UserGroupInformation callerUGI = getUser(req);
boolean checkStart = false;
boolean checkEnd = false;
boolean checkAppTypes = false;
boolean checkAppStates = false;
@ -95,14 +95,12 @@ public AppsInfo getApps(HttpServletRequest req, HttpServletResponse res,
}
if (startedBegin != null && !startedBegin.isEmpty()) {
checkStart = true;
sBegin = Long.parseLong(startedBegin);
if (sBegin < 0) {
throw new BadRequestException("startedTimeBegin must be greater than 0");
}
}
if (startedEnd != null && !startedEnd.isEmpty()) {
checkStart = true;
sEnd = Long.parseLong(startedEnd);
if (sEnd < 0) {
throw new BadRequestException("startedTimeEnd must be greater than 0");
@ -151,6 +149,7 @@ public AppsInfo getApps(HttpServletRequest req, HttpServletResponse res,
final GetApplicationsRequest request =
GetApplicationsRequest.newInstance();
request.setLimit(countNum);
request.setStartRange(new LongRange(sBegin, sEnd));
try {
if (callerUGI == null) {
// TODO: the request should take the params like what RMWebServices does
@ -198,10 +197,6 @@ public Collection<ApplicationReport> run() throws Exception {
continue;
}
if (checkStart
&& (appReport.getStartTime() < sBegin || appReport.getStartTime() > sEnd)) {
continue;
}
if (checkEnd
&& (appReport.getFinishTime() < fBegin || appReport.getFinishTime() > fEnd)) {
continue;