big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-6598. History server getApplicationReport NPE when fetching report for pre-2.8 job (Jason Lowe via jeagles)

Browse Source
This commit is contained in:
Jonathan Eagles 2017-05-15 10:32:01 -05:00
parent 6600abbb5c
commit c48f2976a3
2 changed files with 48 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
View File

@ -330,20 +330,19 @@ private static ApplicationReportExt convertToApplicationReport(
}
if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) {
long vcoreSeconds=Long.parseLong(entityInfo.get(
ApplicationMetricsConstants.APP_CPU_METRICS).toString());
long memorySeconds=Long.parseLong(entityInfo.get(
ApplicationMetricsConstants.APP_MEM_METRICS).toString());
long preemptedMemorySeconds = Long.parseLong(entityInfo.get(
ApplicationMetricsConstants
.APP_MEM_PREEMPT_METRICS).toString());
long preemptedVcoreSeconds = Long.parseLong(entityInfo.get(
ApplicationMetricsConstants
.APP_CPU_PREEMPT_METRICS).toString());
appResources = ApplicationResourceUsageReport
.newInstance(0, 0, null, null, null, memorySeconds, vcoreSeconds, 0,
0, preemptedMemorySeconds, preemptedVcoreSeconds);
long vcoreSeconds = parseLong(entityInfo,
ApplicationMetricsConstants.APP_CPU_METRICS);
long memorySeconds = parseLong(entityInfo,
ApplicationMetricsConstants.APP_MEM_METRICS);
long preemptedMemorySeconds = parseLong(entityInfo,
ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS);
long preemptedVcoreSeconds = parseLong(entityInfo,
ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS);
appResources = ApplicationResourceUsageReport.newInstance(0, 0, null,
null, null, memorySeconds, vcoreSeconds, 0, 0,
preemptedMemorySeconds, preemptedVcoreSeconds);
}
if (entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) {
appTags = new HashSet<String>();
Object obj = entityInfo.get(ApplicationMetricsConstants.APP_TAGS_INFO);
@ -445,6 +444,16 @@ private static ApplicationReportExt convertToApplicationReport(
amNodeLabelExpression), appViewACLs);
}
private static long parseLong(Map<String, Object> entityInfo,
String infoKey) {
long result = 0;
Object infoValue = entityInfo.get(infoKey);
if (infoValue != null) {
result = Long.parseLong(infoValue.toString());
}
return result;
}
private static boolean isFinalState(YarnApplicationState state) {
return state == YarnApplicationState.FINISHED
|| state == YarnApplicationState.FAILED

32
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
View File

@ -143,6 +143,10 @@ private static void prepareTimelineStore(TimelineStore store, int scale)
if (i == 2) {
entities.addEntity(createApplicationTimelineEntity(
appId, true, false, false, true, YarnApplicationState.FINISHED));
} else if (i == 3) {
entities.addEntity(createApplicationTimelineEntity(
appId, false, false, false, false, YarnApplicationState.FINISHED,
true));
} else {
entities.addEntity(createApplicationTimelineEntity(
appId, false, false, false, false, YarnApplicationState.FINISHED));
@ -176,7 +180,7 @@ private static void prepareTimelineStore(TimelineStore store, int scale)
@Test
public void testGetApplicationReport() throws Exception {
for (int i = 1; i <= 2; ++i) {
for (int i = 1; i <= 3; ++i) {
final ApplicationId appId = ApplicationId.newInstance(0, i);
ApplicationReport app;
if (callerUGI == null) {
@ -214,7 +218,7 @@ public ApplicationReport run() throws Exception {
Assert.assertTrue(app.getApplicationTags().contains("Test_APP_TAGS_2"));
// App 2 doesn't have the ACLs, such that the default ACLs " " will be used.
// Nobody except admin and owner has access to the details of the app.
if ((i == 1 && callerUGI != null &&
if ((i != 2 && callerUGI != null &&
callerUGI.getShortUserName().equals("user3")) ||
(i == 2 && callerUGI != null &&
(callerUGI.getShortUserName().equals("user2") ||
@ -245,10 +249,16 @@ public ApplicationReport run() throws Exception {
applicationResourceUsageReport.getMemorySeconds());
Assert
.assertEquals(345, applicationResourceUsageReport.getVcoreSeconds());
Assert.assertEquals(456,
long expectedPreemptMemSecs = 456;
long expectedPreemptVcoreSecs = 789;
if (i == 3) {
expectedPreemptMemSecs = 0;
expectedPreemptVcoreSecs = 0;
}
Assert.assertEquals(expectedPreemptMemSecs,
applicationResourceUsageReport.getPreemptedMemorySeconds());
Assert
.assertEquals(789, applicationResourceUsageReport
.assertEquals(expectedPreemptVcoreSecs, applicationResourceUsageReport
.getPreemptedVcoreSeconds());
Assert.assertEquals(FinalApplicationStatus.UNDEFINED,
app.getFinalApplicationStatus());
@ -486,6 +496,14 @@ private static TimelineEntity createApplicationTimelineEntity(
ApplicationId appId, boolean emptyACLs, boolean noAttemptId,
boolean wrongAppId, boolean enableUpdateEvent,
YarnApplicationState state) {
return createApplicationTimelineEntity(appId, emptyACLs, noAttemptId,
wrongAppId, enableUpdateEvent, state, false);
}
private static TimelineEntity createApplicationTimelineEntity(
ApplicationId appId, boolean emptyACLs, boolean noAttemptId,
boolean wrongAppId, boolean enableUpdateEvent,
YarnApplicationState state, boolean missingPreemptMetrics) {
TimelineEntity entity = new TimelineEntity();
entity.setEntityType(ApplicationMetricsConstants.ENTITY_TYPE);
if (wrongAppId) {
@ -510,8 +528,10 @@ private static TimelineEntity createApplicationTimelineEntity(
Integer.MAX_VALUE + 1L);
entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS, 123);
entityInfo.put(ApplicationMetricsConstants.APP_CPU_METRICS, 345);
entityInfo.put(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS,456);
entityInfo.put(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS,789);
if (!missingPreemptMetrics) {
entityInfo.put(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS, 456);
entityInfo.put(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS, 789);
}
if (emptyACLs) {
entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, "");
} else {