From 571838920d2faca703e3c3417d64123927dfc4ad Mon Sep 17 00:00:00 2001 From: Rohith Sharma K S Date: Tue, 5 Feb 2019 12:39:10 +0530 Subject: [PATCH] YARN-9149. yarn container -status misses logUrl when integrated with ATSv2. Contributed by Abhishek Modi. --- .../hadoop/yarn/client/api/impl/AHSv2ClientImpl.java | 12 ++++++++++-- .../yarn/client/api/impl/TestAHSv2ClientImpl.java | 9 +++++++++ .../util/timeline/TimelineEntityV2Converter.java | 9 ++++++++- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java index e797c28162..19a7546014 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java @@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.client.api.AHSClient; import org.apache.hadoop.yarn.client.api.TimelineReaderClient; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.timeline.TimelineEntityV2Converter; @@ -48,6 +49,7 @@ @InterfaceStability.Unstable public class AHSv2ClientImpl extends AHSClient { private TimelineReaderClient readerClient; + private String logServerUrl; public AHSv2ClientImpl() { super(AHSv2ClientImpl.class.getName()); @@ -55,6 +57,8 @@ public AHSv2ClientImpl() { @Override public void serviceInit(Configuration conf) { + logServerUrl = conf.get( + YarnConfiguration.YARN_LOG_SERVER_URL); readerClient = TimelineReaderClient.createTimelineReaderClient(); readerClient.init(conf); } @@ -119,15 +123,19 @@ public List getApplicationAttempts( @Override public ContainerReport getContainerReport(ContainerId containerId) throws YarnException, IOException { + ApplicationReport appReport = getApplicationReport( + containerId.getApplicationAttemptId().getApplicationId()); TimelineEntity entity = readerClient.getContainerEntity(containerId, "ALL", null); - return TimelineEntityV2Converter.convertToContainerReport(entity); + return TimelineEntityV2Converter.convertToContainerReport( + entity, logServerUrl, appReport.getUser()); } @Override public List getContainers(ApplicationAttemptId applicationAttemptId) throws YarnException, IOException { ApplicationId appId = applicationAttemptId.getApplicationId(); + ApplicationReport appReport = getApplicationReport(appId); Map filters = new HashMap<>(); filters.put("infofilters", "SYSTEM_INFO_PARENT_ENTITY eq {\"id\":\"" + applicationAttemptId.toString() + @@ -140,7 +148,7 @@ public List getContainers(ApplicationAttemptId for (TimelineEntity entity : entities) { ContainerReport container = TimelineEntityV2Converter.convertToContainerReport( - entity); + entity, logServerUrl, appReport.getUser()); containers.add(container); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java index 99473a35d4..1e0a1d6550 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java @@ -61,7 +61,10 @@ public void setup() { Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f); + conf.set(YarnConfiguration.YARN_LOG_SERVER_URL, + "https://localhost:8188/ahs"); client = new AHSv2ClientImpl(); + client.init(conf); spyTimelineReaderClient = mock(TimelineReaderClient.class); client.setReaderClient(spyTimelineReaderClient); } @@ -74,11 +77,17 @@ public void testGetContainerReport() throws IOException, YarnException { final ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1); when(spyTimelineReaderClient.getContainerEntity(containerId, "ALL", null)) .thenReturn(createContainerEntity(containerId)); + when(spyTimelineReaderClient.getApplicationEntity(appId, "ALL", null)) + .thenReturn(createApplicationTimelineEntity(appId, true, false)); ContainerReport report = client.getContainerReport(containerId); Assert.assertEquals(report.getContainerId(), containerId); Assert.assertEquals(report.getAssignedNode().getHost(), "test host"); Assert.assertEquals(report.getAssignedNode().getPort(), 100); Assert.assertEquals(report.getAllocatedResource().getVirtualCores(), 8); + Assert.assertEquals(report.getCreationTime(), 123456); + Assert.assertEquals(report.getLogUrl(), + "https://localhost:8188/ahs/logs/test host:100/" + + "container_0_0001_01_000001/container_0_0001_01_000001/user1"); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineEntityV2Converter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineEntityV2Converter.java index 8d27e3b0ac..98e1e7a7f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineEntityV2Converter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineEntityV2Converter.java @@ -49,6 +49,8 @@ import java.util.NavigableSet; import java.util.Set; +import static org.apache.hadoop.yarn.util.StringHelper.PATH_JOINER; + /** * Utility class to generate reports from timeline entities. */ @@ -57,7 +59,7 @@ private TimelineEntityV2Converter() { } public static ContainerReport convertToContainerReport( - TimelineEntity entity) { + TimelineEntity entity, String serverAddress, String user) { int allocatedMem = 0; int allocatedVcore = 0; String allocatedHost = null; @@ -142,8 +144,13 @@ public static ContainerReport convertToContainerReport( } String logUrl = null; NodeId allocatedNode = null; + String containerId = entity.getId(); if (allocatedHost != null) { allocatedNode = NodeId.newInstance(allocatedHost, allocatedPort); + if (serverAddress != null && user != null) { + logUrl = PATH_JOINER.join(serverAddress, + "logs", allocatedNode, containerId, containerId, user); + } } ContainerReport container = ContainerReport.newInstance( ContainerId.fromString(entity.getId()),