big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-9149. yarn container -status misses logUrl when integrated with ATSv2. Contributed by Abhishek Modi.

Browse Source
This commit is contained in:
Rohith Sharma K S 2019-02-05 12:39:10 +05:30
parent 3efa168e1f
commit 571838920d
3 changed files with 27 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java
View File

@ -31,6 +31,7 @@
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.client.api.AHSClient;
import org.apache.hadoop.yarn.client.api.TimelineReaderClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.timeline.TimelineEntityV2Converter;
@ -48,6 +49,7 @@
@InterfaceStability.Unstable
public class AHSv2ClientImpl extends AHSClient {
private TimelineReaderClient readerClient;
private String logServerUrl;
public AHSv2ClientImpl() {
super(AHSv2ClientImpl.class.getName());
@ -55,6 +57,8 @@ public AHSv2ClientImpl() {
@Override
public void serviceInit(Configuration conf) {
logServerUrl = conf.get(
YarnConfiguration.YARN_LOG_SERVER_URL);
readerClient = TimelineReaderClient.createTimelineReaderClient();
readerClient.init(conf);
}
@ -119,15 +123,19 @@ public List<ApplicationAttemptReport> getApplicationAttempts(
@Override
public ContainerReport getContainerReport(ContainerId containerId)
throws YarnException, IOException {
ApplicationReport appReport = getApplicationReport(
containerId.getApplicationAttemptId().getApplicationId());
TimelineEntity entity = readerClient.getContainerEntity(containerId,
"ALL", null);
return TimelineEntityV2Converter.convertToContainerReport(entity);
return TimelineEntityV2Converter.convertToContainerReport(
entity, logServerUrl, appReport.getUser());
}
@Override
public List<ContainerReport> getContainers(ApplicationAttemptId
applicationAttemptId) throws YarnException, IOException {
ApplicationId appId = applicationAttemptId.getApplicationId();
ApplicationReport appReport = getApplicationReport(appId);
Map<String, String> filters = new HashMap<>();
filters.put("infofilters", "SYSTEM_INFO_PARENT_ENTITY eq {\"id\":\"" +
applicationAttemptId.toString() +
@ -140,7 +148,7 @@ public List<ContainerReport> getContainers(ApplicationAttemptId
for (TimelineEntity entity : entities) {
ContainerReport container =
TimelineEntityV2Converter.convertToContainerReport(
entity);
entity, logServerUrl, appReport.getUser());
containers.add(container);
}
}

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java
View File

@ -61,7 +61,10 @@ public void setup() {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
conf.set(YarnConfiguration.YARN_LOG_SERVER_URL,
"https://localhost:8188/ahs");
client = new AHSv2ClientImpl();
client.init(conf);
spyTimelineReaderClient = mock(TimelineReaderClient.class);
client.setReaderClient(spyTimelineReaderClient);
}
@ -74,11 +77,17 @@ public void testGetContainerReport() throws IOException, YarnException {
final ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
when(spyTimelineReaderClient.getContainerEntity(containerId, "ALL", null))
.thenReturn(createContainerEntity(containerId));
when(spyTimelineReaderClient.getApplicationEntity(appId, "ALL", null))
.thenReturn(createApplicationTimelineEntity(appId, true, false));
ContainerReport report = client.getContainerReport(containerId);
Assert.assertEquals(report.getContainerId(), containerId);
Assert.assertEquals(report.getAssignedNode().getHost(), "test host");
Assert.assertEquals(report.getAssignedNode().getPort(), 100);
Assert.assertEquals(report.getAllocatedResource().getVirtualCores(), 8);
Assert.assertEquals(report.getCreationTime(), 123456);
Assert.assertEquals(report.getLogUrl(),
"https://localhost:8188/ahs/logs/test host:100/"
+ "container_0_0001_01_000001/container_0_0001_01_000001/user1");
}
@Test

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineEntityV2Converter.java
View File

@ -49,6 +49,8 @@
import java.util.NavigableSet;
import java.util.Set;
import static org.apache.hadoop.yarn.util.StringHelper.PATH_JOINER;
/**
* Utility class to generate reports from timeline entities.
*/
@ -57,7 +59,7 @@ private TimelineEntityV2Converter() {
}
public static ContainerReport convertToContainerReport(
TimelineEntity entity) {
TimelineEntity entity, String serverAddress, String user) {
int allocatedMem = 0;
int allocatedVcore = 0;
String allocatedHost = null;
@ -142,8 +144,13 @@ public static ContainerReport convertToContainerReport(
}
String logUrl = null;
NodeId allocatedNode = null;
String containerId = entity.getId();
if (allocatedHost != null) {
allocatedNode = NodeId.newInstance(allocatedHost, allocatedPort);
if (serverAddress != null && user != null) {
logUrl = PATH_JOINER.join(serverAddress,
"logs", allocatedNode, containerId, containerId, user);
}
}
ContainerReport container = ContainerReport.newInstance(
ContainerId.fromString(entity.getId()),