big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-4452. NPE when submit Unmanaged application. Contributed by Naganarasimha G R.

Browse Source
This commit is contained in:
Junping Du 2015-12-16 10:57:39 -08:00
parent 607473e1d0
commit 50bd067e1d
3 changed files with 51 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
hadoop-yarn-project/CHANGES.txt
View File

@ -1138,6 +1138,9 @@ Release 2.8.0 - UNRELEASED
YARN-4440. FSAppAttempt#getAllowedLocalityLevelByTime should init the
lastScheduler time. (Lin Yiqun via zxu)
YARN-4452. NPE when submit Unmanaged application. (Naganarasimha G R
via junping_du)
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
@ -1186,6 +1189,9 @@ Release 2.7.3 - UNRELEASED
YARN-4439. Clarify NMContainerStatus#toString method. (Jian He via xgong)
YARN-4452. NPE when submit Unmanaged application. (Naganarasimha G R via
junping_du)
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
@ -2040,6 +2046,9 @@ Release 2.6.4 - UNRELEASED
YARN-3535. Scheduler must re-request container resources when RMContainer transitions
from ALLOCATED to KILLED (rohithsharma and peng.zhang via asuresh)
YARN-4452. NPE when submit Unmanaged application. (Naganarasimha G R
via junping_du)
Release 2.6.3 - UNRELEASED
INCOMPATIBLE CHANGES

22
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
View File

@ -160,6 +160,8 @@ public void appACLsUpdated(RMApp app, String appViewACLs,
public void appAttemptRegistered(RMAppAttempt appAttempt,
long registeredTime) {
if (publishSystemMetrics) {
ContainerId container = (appAttempt.getMasterContainer() == null) ? null
: appAttempt.getMasterContainer().getId();
dispatcher.getEventHandler().handle(
new AppAttemptRegisteredEvent(
appAttempt.getAppAttemptId(),
@ -167,7 +169,7 @@ public void appAttemptRegistered(RMAppAttempt appAttempt,
appAttempt.getRpcPort(),
appAttempt.getTrackingUrl(),
appAttempt.getOriginalTrackingUrl(),
appAttempt.getMasterContainer().getId(),
container,
registeredTime));
}
}
@ -176,6 +178,8 @@ public void appAttemptRegistered(RMAppAttempt appAttempt,
public void appAttemptFinished(RMAppAttempt appAttempt,
RMAppAttemptState appAttemtpState, RMApp app, long finishedTime) {
if (publishSystemMetrics) {
ContainerId container = (appAttempt.getMasterContainer() == null) ? null
: appAttempt.getMasterContainer().getId();
dispatcher.getEventHandler().handle(
new AppAttemptFinishedEvent(
appAttempt.getAppAttemptId(),
@ -187,7 +191,7 @@ public void appAttemptFinished(RMAppAttempt appAttempt,
app.getFinalApplicationStatus(),
RMServerUtils.createApplicationAttemptState(appAttemtpState),
finishedTime,
appAttempt.getMasterContainer().getId()));
container));
}
}
@ -390,9 +394,10 @@ private static TimelineEntity createApplicationEntity(
event.getHost());
eventInfo.put(AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO,
event.getRpcPort());
eventInfo.put(
AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO,
event.getMasterContainerId().toString());
if (event.getMasterContainerId() != null) {
eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO,
event.getMasterContainerId().toString());
}
tEvent.setEventInfo(eventInfo);
entity.addEvent(tEvent);
putEntity(entity);
@ -417,9 +422,10 @@ private void publishAppAttemptFinishedEvent(AppAttemptFinishedEvent event) {
event.getFinalApplicationStatus().toString());
eventInfo.put(AppAttemptMetricsConstants.STATE_EVENT_INFO,
event.getYarnApplicationAttemptState().toString());
eventInfo.put(
AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO,
event.getMasterContainerId().toString());
if (event.getMasterContainerId() != null) {
eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO,
event.getMasterContainerId().toString());
}
tEvent.setEventInfo(eventInfo);
entity.addEvent(tEvent);
putEntity(entity);

34
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
View File

@ -256,11 +256,31 @@ public void testPublishApplicationMetrics() throws Exception {
}
}
@Test(timeout = 10000)
public void testPublishAppAttemptMetricsForUnmanagedAM() throws Exception {
ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1);
RMAppAttempt appAttempt = createRMAppAttempt(appAttemptId,true);
metricsPublisher.appAttemptRegistered(appAttempt, Integer.MAX_VALUE + 1L);
RMApp app = mock(RMApp.class);
when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.UNDEFINED);
metricsPublisher.appAttemptFinished(appAttempt, RMAppAttemptState.FINISHED, app,
Integer.MAX_VALUE + 2L);
TimelineEntity entity = null;
do {
entity =
store.getEntity(appAttemptId.toString(),
AppAttemptMetricsConstants.ENTITY_TYPE,
EnumSet.allOf(Field.class));
// ensure two events are both published before leaving the loop
} while (entity == null || entity.getEvents().size() < 2);
}
@Test(timeout = 10000)
public void testPublishAppAttemptMetrics() throws Exception {
ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1);
RMAppAttempt appAttempt = createRMAppAttempt(appAttemptId);
RMAppAttempt appAttempt = createRMAppAttempt(appAttemptId, false);
metricsPublisher.appAttemptRegistered(appAttempt, Integer.MAX_VALUE + 1L);
RMApp app = mock(RMApp.class);
when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.UNDEFINED);
@ -435,15 +455,17 @@ private static RMApp createRMApp(ApplicationId appId) {
}
private static RMAppAttempt createRMAppAttempt(
ApplicationAttemptId appAttemptId) {
ApplicationAttemptId appAttemptId, boolean unmanagedAMAttempt) {
RMAppAttempt appAttempt = mock(RMAppAttempt.class);
when(appAttempt.getAppAttemptId()).thenReturn(appAttemptId);
when(appAttempt.getHost()).thenReturn("test host");
when(appAttempt.getRpcPort()).thenReturn(-100);
Container container = mock(Container.class);
when(container.getId())
.thenReturn(ContainerId.newContainerId(appAttemptId, 1));
when(appAttempt.getMasterContainer()).thenReturn(container);
if (!unmanagedAMAttempt) {
Container container = mock(Container.class);
when(container.getId())
.thenReturn(ContainerId.newContainerId(appAttemptId, 1));
when(appAttempt.getMasterContainer()).thenReturn(container);
}
when(appAttempt.getDiagnostics()).thenReturn("test diagnostics info");
when(appAttempt.getTrackingUrl()).thenReturn("test tracking url");
when(appAttempt.getOriginalTrackingUrl()).thenReturn(