big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-9418. ATSV2 /apps//entities/YARN_CONTAINER rest api does not show metrics. Contributed by Prabhu Joseph.

Browse Source
This commit is contained in:
Giovanni Matteo Fumarola 2019-04-01 11:06:51 -07:00
parent b0d24ef39c
commit 332cab5518
4 changed files with 19 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
View File

@ -186,8 +186,6 @@ public void reportContainerResourceUsage(Container container, Long pmemUsage,
Math.round(cpuUsagePercentPerCore));
entity.addMetric(cpuMetric);
}
entity.setIdPrefix(TimelineServiceHelper.
invertLong(container.getContainerStartTime()));
ApplicationId appId = container.getContainerId().
getApplicationAttemptId().getApplicationId();
try {
@ -249,7 +247,6 @@ private void publishContainerCreatedEvent(ContainerEvent event) {
long containerStartTime = container.getContainerStartTime();
entity.addEvent(tEvent);
entity.setCreatedTime(containerStartTime);
entity.setIdPrefix(TimelineServiceHelper.invertLong(containerStartTime));
dispatcher.getEventHandler().handle(new TimelinePublishEvent(entity,
containerId.getApplicationAttemptId().getApplicationId()));
}
@ -277,7 +274,6 @@ private void publishContainerFinishedEvent(ContainerStatus containerStatus,
tEvent.setId(ContainerMetricsConstants.FINISHED_EVENT_TYPE);
tEvent.setTimestamp(containerFinishTime);
entity.addEvent(tEvent);
entity.setIdPrefix(TimelineServiceHelper.invertLong(containerStartTime));
dispatcher.getEventHandler().handle(new TimelinePublishEvent(entity,
containerId.getApplicationAttemptId().getApplicationId()));
@ -295,8 +291,6 @@ private void publishContainerLocalizationEvent(
tEvent.setId(eventType);
tEvent.setTimestamp(event.getTimestamp());
entity.addEvent(tEvent);
entity.setIdPrefix(TimelineServiceHelper.
invertLong(container.getContainerStartTime()));
ApplicationId appId = container.getContainerId().
getApplicationAttemptId().getApplicationId();
@ -328,6 +322,8 @@ private static ContainerEntity createContainerEntity(
ContainerId containerId) {
ContainerEntity entity = new ContainerEntity();
entity.setId(containerId.toString());
entity.setIdPrefix(TimelineServiceHelper.invertLong(
containerId.getContainerId()));
Identifier parentIdentifier = new Identifier();
parentIdentifier
.setType(TimelineEntityType.YARN_APPLICATION_ATTEMPT.name());

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java
View File

@ -46,6 +46,7 @@
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationContainerFinishedEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
import org.apache.hadoop.yarn.util.TimelineServiceHelper;
import org.junit.Assert;
import org.junit.Test;
import org.junit.After;
@ -140,6 +141,8 @@ private Context createMockContext() {
entity.getInfo().get(ContainerMetricsConstants.DIAGNOSTICS_INFO));
Assert.assertEquals(exitStatus,
entity.getInfo().get(ContainerMetricsConstants.EXIT_STATUS_INFO));
Assert.assertEquals(TimelineServiceHelper.invertLong(
cId.getContainerId()), entity.getIdPrefix());
}
@Test public void testContainerResourceUsage() {
@ -148,29 +151,31 @@ private Context createMockContext() {
Container aContainer = mock(Container.class);
when(aContainer.getContainerId()).thenReturn(ContainerId
.newContainerId(ApplicationAttemptId.newInstance(appId, 1), 0L));
long idPrefix = TimelineServiceHelper.invertLong(
aContainer.getContainerId().getContainerId());
publisher.reportContainerResourceUsage(aContainer, 1024L, 8F);
verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 8);
verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 8, idPrefix);
timelineClient.reset();
publisher.reportContainerResourceUsage(aContainer, 1024L, 0.8F);
verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 1);
verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 1, idPrefix);
timelineClient.reset();
publisher.reportContainerResourceUsage(aContainer, 1024L, 0.49F);
verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 0);
verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 0, idPrefix);
timelineClient.reset();
publisher.reportContainerResourceUsage(aContainer, 1024L,
(float) ResourceCalculatorProcessTree.UNAVAILABLE);
verifyPublishedResourceUsageMetrics(timelineClient, 1024L,
ResourceCalculatorProcessTree.UNAVAILABLE);
ResourceCalculatorProcessTree.UNAVAILABLE, idPrefix);
}
private void verifyPublishedResourceUsageMetrics(
DummyTimelineClient timelineClient, long memoryUsage, int cpuUsage) {
private void verifyPublishedResourceUsageMetrics(DummyTimelineClient
dummyTimelineClient, long memoryUsage, int cpuUsage, long idPrefix) {
TimelineEntity[] entities = null;
for (int i = 0; i < 10; i++) {
entities = timelineClient.getLastPublishedEntities();
entities = dummyTimelineClient.getLastPublishedEntities();
if (entities != null) {
break;
}
@ -188,6 +193,7 @@ private void verifyPublishedResourceUsageMetrics(
assertNotNull("entities are expected to be published", entities);
assertEquals("Expected number of metrics notpublished",
numberOfResourceMetrics, entities[0].getMetrics().size());
assertEquals(idPrefix, entities[0].getIdPrefix());
Iterator<TimelineMetric> metrics = entities[0].getMetrics().iterator();
while (metrics.hasNext()) {
TimelineMetric metric = metrics.next();

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
View File

@ -458,6 +458,8 @@ private static ContainerEntity createContainerEntity(
ContainerId containerId) {
ContainerEntity entity = new ContainerEntity();
entity.setId(containerId.toString());
entity.setIdPrefix(TimelineServiceHelper.invertLong(
containerId.getContainerId()));
entity.setParent(new Identifier(TimelineEntityType.YARN_APPLICATION_ATTEMPT
.name(), containerId.getApplicationAttemptId().toString()));
return entity;

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
View File

@ -289,7 +289,8 @@ public void testPublishContainerMetrics() throws Exception {
File appFile = new File(outputDirApp, timelineServiceFileName);
Assert.assertTrue(appFile.exists());
verifyEntity(appFile, 2,
ContainerMetricsConstants.CREATED_IN_RM_EVENT_TYPE, 0, 0);
ContainerMetricsConstants.CREATED_IN_RM_EVENT_TYPE, 0,
TimelineServiceHelper.invertLong(containerId.getContainerId()));
}
private RMApp createAppAndRegister(ApplicationId appId) {