big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-3701. Isolating the error of generating a single app report when

Browse Source 
getting all apps from generic history service. Contributed by Zhijie
Shen
This commit is contained in:
Xuan 2015-05-22 14:34:39 -07:00
parent f34638399a
commit 455b3acf0e
3 changed files with 50 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -550,6 +550,9 @@ Release 2.7.1 - UNRELEASED
YARN-3675. FairScheduler: RM quits when node removal races with
continuous-scheduling on the same node. (Anubhav Dhoot via kasha)
YARN-3701. Isolating the error of generating a single app report when
getting all apps from generic history service. (Zhijie Shen via xgong)
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES

37
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
View File

@ -25,6 +25,8 @@
import java.util.List;
import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AuthorizationException;
@ -67,6 +69,8 @@
public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
implements
ApplicationHistoryManager {
private static final Log LOG = LogFactory
.getLog(ApplicationHistoryManagerOnTimelineStore.class);
@VisibleForTesting
static final String UNAVAILABLE = "N/A";
@ -107,9 +111,14 @@ public Map<ApplicationId, ApplicationReport> getAllApplications()
new LinkedHashMap<ApplicationId, ApplicationReport>();
if (entities != null && entities.getEntities() != null) {
for (TimelineEntity entity : entities.getEntities()) {
ApplicationReportExt app =
generateApplicationReport(entity, ApplicationReportField.ALL);
apps.put(app.appReport.getApplicationId(), app.appReport);
try {
ApplicationReportExt app =
generateApplicationReport(entity, ApplicationReportField.ALL);
apps.put(app.appReport.getApplicationId(), app.appReport);
} catch (Exception e) {
LOG.error("Error on generating application report for " +
entity.getEntityId(), e);
}
}
}
return apps;
@ -142,9 +151,18 @@ public Map<ApplicationId, ApplicationReport> getAllApplications()
@Override
public ApplicationAttemptReport getApplicationAttempt(
ApplicationAttemptId appAttemptId) throws YarnException, IOException {
ApplicationReportExt app = getApplication(
appAttemptId.getApplicationId(), ApplicationReportField.USER_AND_ACLS);
checkAccess(app);
return getApplicationAttempt(appAttemptId, true);
}
private ApplicationAttemptReport getApplicationAttempt(
ApplicationAttemptId appAttemptId, boolean checkACLs)
throws YarnException, IOException {
if (checkACLs) {
ApplicationReportExt app = getApplication(
appAttemptId.getApplicationId(),
ApplicationReportField.USER_AND_ACLS);
checkAccess(app);
}
TimelineEntity entity = timelineDataManager.getEntity(
AppAttemptMetricsConstants.ENTITY_TYPE,
appAttemptId.toString(), EnumSet.allOf(Field.class),
@ -182,7 +200,8 @@ public ContainerReport getContainer(ContainerId containerId)
@Override
public ContainerReport getAMContainer(ApplicationAttemptId appAttemptId)
throws YarnException, IOException {
ApplicationAttemptReport appAttempt = getApplicationAttempt(appAttemptId);
ApplicationAttemptReport appAttempt =
getApplicationAttempt(appAttemptId, false);
return getContainer(appAttempt.getAMContainerId());
}
@ -517,8 +536,8 @@ private ApplicationReportExt generateApplicationReport(TimelineEntity entity,
try {
checkAccess(app);
if (app.appReport.getCurrentApplicationAttemptId() != null) {
ApplicationAttemptReport appAttempt =
getApplicationAttempt(app.appReport.getCurrentApplicationAttemptId());
ApplicationAttemptReport appAttempt = getApplicationAttempt(
app.appReport.getCurrentApplicationAttemptId(), false);
app.appReport.setHost(appAttempt.getHost());
app.appReport.setRpcPort(appAttempt.getRpcPort());
app.appReport.setTrackingUrl(appAttempt.getTrackingUrl());

25
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
View File

@ -78,7 +78,9 @@ public static void prepareStore() throws Exception {
store = createStore(SCALE);
TimelineEntities entities = new TimelineEntities();
entities.addEntity(createApplicationTimelineEntity(
ApplicationId.newInstance(0, SCALE + 1), true, false));
ApplicationId.newInstance(0, SCALE + 1), true, true, false));
entities.addEntity(createApplicationTimelineEntity(
ApplicationId.newInstance(0, SCALE + 2), true, false, true));
store.put(entities);
}
@ -133,9 +135,11 @@ private static void prepareTimelineStore(TimelineStore store, int scale)
TimelineEntities entities = new TimelineEntities();
ApplicationId appId = ApplicationId.newInstance(0, i);
if (i == 2) {
entities.addEntity(createApplicationTimelineEntity(appId, true, true));
entities.addEntity(createApplicationTimelineEntity(
appId, true, false, false));
} else {
entities.addEntity(createApplicationTimelineEntity(appId, false, true));
entities.addEntity(createApplicationTimelineEntity(
appId, false, false, false));
}
store.put(entities);
for (int j = 1; j <= scale; ++j) {
@ -334,6 +338,10 @@ public void testGetApplications() throws Exception {
historyManager.getAllApplications().values();
Assert.assertNotNull(apps);
Assert.assertEquals(SCALE + 1, apps.size());
ApplicationId ignoredAppId = ApplicationId.newInstance(0, SCALE + 2);
for (ApplicationReport app : apps) {
Assert.assertNotEquals(ignoredAppId, app.getApplicationId());
}
}
@Test
@ -433,10 +441,15 @@ public ContainerReport run() throws Exception {
}
private static TimelineEntity createApplicationTimelineEntity(
ApplicationId appId, boolean emptyACLs, boolean noAttempt) {
ApplicationId appId, boolean emptyACLs, boolean noAttemptId,
boolean wrongAppId) {
TimelineEntity entity = new TimelineEntity();
entity.setEntityType(ApplicationMetricsConstants.ENTITY_TYPE);
entity.setEntityId(appId.toString());
if (wrongAppId) {
entity.setEntityId("wrong_app_id");
} else {
entity.setEntityId(appId.toString());
}
entity.setDomainId(TimelineDataManager.DEFAULT_DOMAIN_ID);
entity.addPrimaryFilter(
TimelineStore.SystemFilter.ENTITY_OWNER.toString(), "yarn");
@ -472,7 +485,7 @@ private static TimelineEntity createApplicationTimelineEntity(
FinalApplicationStatus.UNDEFINED.toString());
eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO,
YarnApplicationState.FINISHED.toString());
if (noAttempt) {
if (!noAttemptId) {
eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO,
ApplicationAttemptId.newInstance(appId, 1));
}