big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-3393. Getting application(s) goes wrong when app finishes before

Browse Source 
starting the attempt. Contributed by Zhijie Shen
This commit is contained in:
Xuan 2015-03-23 20:33:16 -07:00
parent 2c238ae4e0
commit 9fae455e26
3 changed files with 42 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -828,6 +828,9 @@ Release 2.7.0 - UNRELEASED
YARN-3336. FileSystem memory leak in DelegationTokenRenewer.
(Zhihai Xu via cnauroth)
YARN-3393. Getting application(s) goes wrong when app finishes before
starting the attempt. (Zhijie Shen via xgong)
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
View File

@ -517,15 +517,14 @@ private ApplicationReportExt generateApplicationReport(TimelineEntity entity,
if (app.appReport.getCurrentApplicationAttemptId() != null) {
ApplicationAttemptReport appAttempt =
getApplicationAttempt(app.appReport.getCurrentApplicationAttemptId());
if (appAttempt != null) {
app.appReport.setHost(appAttempt.getHost());
app.appReport.setRpcPort(appAttempt.getRpcPort());
app.appReport.setTrackingUrl(appAttempt.getTrackingUrl());
app.appReport.setOriginalTrackingUrl(appAttempt.getOriginalTrackingUrl());
}
app.appReport.setHost(appAttempt.getHost());
app.appReport.setRpcPort(appAttempt.getRpcPort());
app.appReport.setTrackingUrl(appAttempt.getTrackingUrl());
app.appReport.setOriginalTrackingUrl(appAttempt.getOriginalTrackingUrl());
}
} catch (AuthorizationException e) {
} catch (AuthorizationException | ApplicationAttemptNotFoundException e) {
// AuthorizationException is thrown because the user doesn't have access
// It's possible that the app is finished before the first attempt is created.
app.appReport.setDiagnostics(null);
app.appReport.setCurrentApplicationAttemptId(null);
}

39
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
View File

@ -76,6 +76,10 @@ public class TestApplicationHistoryManagerOnTimelineStore {
@BeforeClass
public static void prepareStore() throws Exception {
store = createStore(SCALE);
TimelineEntities entities = new TimelineEntities();
entities.addEntity(createApplicationTimelineEntity(
ApplicationId.newInstance(0, SCALE + 1), true, false));
store.put(entities);
}
public static TimelineStore createStore(int scale) throws Exception {
@ -129,9 +133,9 @@ private static void prepareTimelineStore(TimelineStore store, int scale)
TimelineEntities entities = new TimelineEntities();
ApplicationId appId = ApplicationId.newInstance(0, i);
if (i == 2) {
entities.addEntity(createApplicationTimelineEntity(appId, true));
entities.addEntity(createApplicationTimelineEntity(appId, true, true));
} else {
entities.addEntity(createApplicationTimelineEntity(appId, false));
entities.addEntity(createApplicationTimelineEntity(appId, false, true));
}
store.put(entities);
for (int j = 1; j <= scale; ++j) {
@ -215,6 +219,27 @@ public ApplicationReport run() throws Exception {
}
}
@Test
public void testGetApplicationReportWithNotAttempt() throws Exception {
final ApplicationId appId = ApplicationId.newInstance(0, SCALE + 1);
ApplicationReport app;
if (callerUGI == null) {
app = historyManager.getApplication(appId);
} else {
app =
callerUGI.doAs(new PrivilegedExceptionAction<ApplicationReport> () {
@Override
public ApplicationReport run() throws Exception {
return historyManager.getApplication(appId);
}
});
}
Assert.assertNotNull(app);
Assert.assertEquals(appId, app.getApplicationId());
Assert.assertEquals(ApplicationAttemptId.newInstance(appId, -1),
app.getCurrentApplicationAttemptId());
}
@Test
public void testGetApplicationAttemptReport() throws Exception {
final ApplicationAttemptId appAttemptId =
@ -308,7 +333,7 @@ public void testGetApplications() throws Exception {
Collection<ApplicationReport> apps =
historyManager.getAllApplications().values();
Assert.assertNotNull(apps);
Assert.assertEquals(SCALE, apps.size());
Assert.assertEquals(SCALE + 1, apps.size());
}
@Test
@ -408,7 +433,7 @@ public ContainerReport run() throws Exception {
}
private static TimelineEntity createApplicationTimelineEntity(
ApplicationId appId, boolean emptyACLs) {
ApplicationId appId, boolean emptyACLs, boolean noAttempt) {
TimelineEntity entity = new TimelineEntity();
entity.setEntityType(ApplicationMetricsConstants.ENTITY_TYPE);
entity.setEntityId(appId.toString());
@ -447,8 +472,10 @@ private static TimelineEntity createApplicationTimelineEntity(
FinalApplicationStatus.UNDEFINED.toString());
eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO,
YarnApplicationState.FINISHED.toString());
eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO,
ApplicationAttemptId.newInstance(appId, 1));
if (noAttempt) {
eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO,
ApplicationAttemptId.newInstance(appId, 1));
}
tEvent.setEventInfo(eventInfo);
entity.addEvent(tEvent);
return entity;