big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-9403.GET /apps/{appid}/entities/YARN_APPLICATION accesses application table instead of entity table (#4516)

Browse Source 
Co-authored-by: Ashutosh Gupta <ashugpt@amazon.com>
This commit is contained in:
Ashutosh Gupta 2022-07-02 17:29:28 +01:00 committed by GitHub
parent 57cbde9abf
commit 151bb31c47
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 142 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
View File

@ -2610,4 +2610,20 @@ public void testGetAppsMetricsRange() throws Exception {
client.destroy();
}
}
@Test
public void testGetEntityWithSystemEntityType() throws Exception {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + getServerPort() + "/ws/v2/" +
"timeline/apps/application_1111111111_1111/" +
"entities/YARN_APPLICATION");
ClientResponse resp = getResponse(client, uri);
Set<TimelineEntity> entities =
resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
assertEquals(0, entities.size());
} finally {
client.destroy();
}
}
}

52
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java
View File

@ -43,18 +43,20 @@ public static TimelineEntityReader createSingleEntityReader(
TimelineReaderContext context, TimelineDataToRetrieve dataToRetrieve) {
// currently the types that are handled separate from the generic entity
// table are application, flow run, and flow activity entities
if (TimelineEntityType.YARN_APPLICATION.matches(context.getEntityType())) {
return new ApplicationEntityReader(context, dataToRetrieve);
} else if (TimelineEntityType.
YARN_FLOW_RUN.matches(context.getEntityType())) {
return new FlowRunEntityReader(context, dataToRetrieve);
} else if (TimelineEntityType.
YARN_FLOW_ACTIVITY.matches(context.getEntityType())) {
return new FlowActivityEntityReader(context, dataToRetrieve);
} else {
// assume we're dealing with a generic entity read
return new GenericEntityReader(context, dataToRetrieve);
if (!context.isGenericEntity()) {
if (TimelineEntityType.
YARN_APPLICATION.matches(context.getEntityType())) {
return new ApplicationEntityReader(context, dataToRetrieve);
} else if (TimelineEntityType.
YARN_FLOW_RUN.matches(context.getEntityType())) {
return new FlowRunEntityReader(context, dataToRetrieve);
} else if (TimelineEntityType.
YARN_FLOW_ACTIVITY.matches(context.getEntityType())) {
return new FlowActivityEntityReader(context, dataToRetrieve);
}
}
// assume we're dealing with a generic entity read
return new GenericEntityReader(context, dataToRetrieve);
}
/**
@ -73,21 +75,23 @@ public static TimelineEntityReader createMultipleEntitiesReader(
TimelineDataToRetrieve dataToRetrieve) {
// currently the types that are handled separate from the generic entity
// table are application, flow run, and flow activity entities
if (TimelineEntityType.YARN_APPLICATION.matches(context.getEntityType())) {
return new ApplicationEntityReader(context, filters, dataToRetrieve);
} else if (TimelineEntityType.
YARN_FLOW_ACTIVITY.matches(context.getEntityType())) {
return new FlowActivityEntityReader(context, filters, dataToRetrieve);
} else if (TimelineEntityType.
YARN_FLOW_RUN.matches(context.getEntityType())) {
return new FlowRunEntityReader(context, filters, dataToRetrieve);
} else {
if (context.getDoAsUser() != null) {
return new SubApplicationEntityReader(context, filters, dataToRetrieve);
if (!context.isGenericEntity()) {
if (TimelineEntityType.
YARN_APPLICATION.matches(context.getEntityType())) {
return new ApplicationEntityReader(context, filters, dataToRetrieve);
} else if (TimelineEntityType.
YARN_FLOW_ACTIVITY.matches(context.getEntityType())) {
return new FlowActivityEntityReader(context, filters, dataToRetrieve);
} else if (TimelineEntityType.
YARN_FLOW_RUN.matches(context.getEntityType())) {
return new FlowRunEntityReader(context, filters, dataToRetrieve);
}
// assume we're dealing with a generic entity read
return new GenericEntityReader(context, filters, dataToRetrieve);
}
if (context.getDoAsUser() != null) {
return new SubApplicationEntityReader(context, filters, dataToRetrieve);
}
// assume we're dealing with a generic entity read
return new GenericEntityReader(context, filters, dataToRetrieve);
}
/**

23
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderContext.java
View File

@ -33,6 +33,8 @@ public class TimelineReaderContext extends TimelineContext {
private String entityId;
private Long entityIdPrefix;
private String doAsUser;
private boolean genericEntity = false;
public TimelineReaderContext(String clusterId, String userId, String flowName,
Long flowRunId, String appId, String entityType, String entityId) {
super(clusterId, userId, flowName, flowRunId, appId);
@ -55,10 +57,19 @@ public TimelineReaderContext(String clusterId, String userId, String flowName,
this.doAsUser = doasUser;
}
public TimelineReaderContext(String clusterId, String userId, String flowName,
Long flowRunId, String appId, String entityType, Long entityIdPrefix,
String entityId, String doasUser, boolean genericEntity) {
this(clusterId, userId, flowName, flowRunId, appId, entityType,
entityIdPrefix, entityId, doasUser);
this.genericEntity = genericEntity;
}
public TimelineReaderContext(TimelineReaderContext other) {
this(other.getClusterId(), other.getUserId(), other.getFlowName(),
other.getFlowRunId(), other.getAppId(), other.getEntityType(),
other.getEntityIdPrefix(), other.getEntityId(), other.getDoAsUser());
other.getEntityIdPrefix(), other.getEntityId(), other.getDoAsUser(),
other.genericEntity);
}
@Override
@ -130,4 +141,14 @@ public String getDoAsUser() {
public void setDoAsUser(String doAsUser) {
this.doAsUser = doAsUser;
}
public boolean isGenericEntity() {
return genericEntity;
}
public void setGenericEntity(boolean genericEntity) {
this.genericEntity = genericEntity;
}
}

120
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
View File

@ -371,6 +371,7 @@ public Set<TimelineEntity> getEntities(
}
context.setEntityType(
TimelineReaderWebServicesUtils.parseStr(entityType));
context.setGenericEntity(true);
entities = timelineReaderManager.getEntities(context,
TimelineReaderWebServicesUtils.createTimelineEntityFilters(
limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo,
@ -516,7 +517,7 @@ public Set<TimelineEntity> getEntities(
flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo,
isRelatedTo, infofilters, conffilters, metricfilters, eventfilters,
confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
metricsTimeStart, metricsTimeEnd, fromId);
metricsTimeStart, metricsTimeEnd, fromId, true);
}
/**
@ -636,6 +637,23 @@ public Set<TimelineEntity> getEntities(
@QueryParam("metricstimestart") String metricsTimeStart,
@QueryParam("metricstimeend") String metricsTimeEnd,
@QueryParam("fromid") String fromId) {
return getEntities(req, res, null, appId, entityType, userId, flowName,
flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo,
isRelatedTo, infofilters, conffilters, metricfilters, eventfilters,
confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
metricsTimeStart, metricsTimeEnd, fromId, true);
}
public Set<TimelineEntity> getEntities(HttpServletRequest req,
HttpServletResponse res, String clusterId, String appId,
String entityType, String userId, String flowName,
String flowRunId, String limit, String createdTimeStart,
String createdTimeEnd, String relatesTo, String isRelatedTo,
String infofilters, String conffilters, String metricfilters,
String eventfilters, String confsToRetrieve, String metricsToRetrieve,
String fields, String metricsLimit, String metricsTimeStart,
String metricsTimeEnd, String fromId,
boolean genericEntity) {
String url = req.getRequestURI() +
(req.getQueryString() == null ? "" :
QUERY_STRING_SEP + req.getQueryString());
@ -652,6 +670,7 @@ public Set<TimelineEntity> getEntities(
TimelineReaderContext context = TimelineReaderWebServicesUtils
.createTimelineReaderContext(clusterId, userId, flowName, flowRunId,
appId, entityType, null, null);
context.setGenericEntity(genericEntity);
entities = timelineReaderManager.getEntities(context,
TimelineReaderWebServicesUtils
.createTimelineEntityFilters(limit, createdTimeStart,
@ -777,6 +796,54 @@ public TimelineEntity getEntity(
return entity;
}
public TimelineEntity getEntity(HttpServletRequest req,
HttpServletResponse res, String clusterId, String appId,
String entityType, String entityId, String userId, String flowName,
String flowRunId, String confsToRetrieve, String metricsToRetrieve,
String fields, String metricsLimit, String metricsTimeStart,
String metricsTimeEnd, String entityIdPrefix,
boolean genericEntity) {
String url = req.getRequestURI() +
(req.getQueryString() == null ? "" :
QUERY_STRING_SEP + req.getQueryString());
UserGroupInformation callerUGI =
TimelineReaderWebServicesUtils.getUser(req);
LOG.info("Received URL " + url + " from user " +
TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow();
boolean succeeded = false;
init(res);
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
TimelineEntity entity = null;
TimelineReaderContext context = TimelineReaderWebServicesUtils.
createTimelineReaderContext(clusterId, userId, flowName, flowRunId,
appId, entityType, entityIdPrefix, entityId);
context.setGenericEntity(genericEntity);
try {
entity = timelineReaderManager.getEntity(context,
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
metricsTimeStart, metricsTimeEnd));
checkAccessForGenericEntity(entity, callerUGI);
succeeded = true;
} catch (Exception e) {
handleException(e, url, startTime, "Either flowrunid or metricslimit or"
+ " metricstime start/end");
} finally {
long latency = Time.monotonicNow() - startTime;
METRICS.addGetEntitiesLatency(latency, succeeded);
LOG.info("Processed URL " + url +
" (Took " + latency + " ms.)");
}
if (entity == null) {
LOG.info("Processed URL " + url + " but entity not found" + " (Took " +
(Time.monotonicNow() - startTime) + " ms.)");
throw new NotFoundException("Timeline entity {id: " + entityId +
", type: " + entityType + " } is not found");
}
return entity;
}
/**
* Return a single entity of the given entity type and Id. Cluster ID is not
* provided by client so default cluster ID has to be taken. If userid, flow
@ -853,7 +920,7 @@ public TimelineEntity getEntity(
@QueryParam("entityidprefix") String entityIdPrefix) {
return getEntity(req, res, null, appId, entityType, entityId, userId,
flowName, flowRunId, confsToRetrieve, metricsToRetrieve, fields,
metricsLimit, metricsTimeStart, metricsTimeEnd, entityIdPrefix);
metricsLimit, metricsTimeStart, metricsTimeEnd, entityIdPrefix, true);
}
/**
@ -932,44 +999,9 @@ public TimelineEntity getEntity(
@QueryParam("metricstimestart") String metricsTimeStart,
@QueryParam("metricstimeend") String metricsTimeEnd,
@QueryParam("entityidprefix") String entityIdPrefix) {
String url = req.getRequestURI() +
(req.getQueryString() == null ? "" :
QUERY_STRING_SEP + req.getQueryString());
UserGroupInformation callerUGI =
TimelineReaderWebServicesUtils.getUser(req);
LOG.info("Received URL " + url + " from user " +
TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow();
boolean succeeded = false;
init(res);
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
TimelineEntity entity = null;
try {
entity = timelineReaderManager.getEntity(
TimelineReaderWebServicesUtils.createTimelineReaderContext(
clusterId, userId, flowName, flowRunId, appId, entityType,
entityIdPrefix, entityId),
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
metricsTimeStart, metricsTimeEnd));
checkAccessForGenericEntity(entity, callerUGI);
succeeded = true;
} catch (Exception e) {
handleException(e, url, startTime, "Either flowrunid or metricslimit or"
+ " metricstime start/end");
} finally {
long latency = Time.monotonicNow() - startTime;
METRICS.addGetEntitiesLatency(latency, succeeded);
LOG.info("Processed URL " + url +
" (Took " + latency + " ms.)");
}
if (entity == null) {
LOG.info("Processed URL " + url + " but entity not found" + " (Took " +
(Time.monotonicNow() - startTime) + " ms.)");
throw new NotFoundException("Timeline entity {id: " + entityId +
", type: " + entityType + " } is not found");
}
return entity;
return getEntity(req, res, clusterId, appId, entityType, entityId,
userId, flowName, flowRunId, confsToRetrieve, metricsToRetrieve, fields,
metricsLimit, metricsTimeStart, metricsTimeEnd, entityIdPrefix, true);
}
/**
@ -2088,7 +2120,7 @@ public Set<TimelineEntity> getFlowRunApps(
flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo,
isRelatedTo, infofilters, conffilters, metricfilters, eventfilters,
confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
metricsTimeStart, metricsTimeEnd, fromId);
metricsTimeStart, metricsTimeEnd, fromId, false);
}
/**
@ -2202,7 +2234,7 @@ public Set<TimelineEntity> getFlowRunApps(
flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo,
isRelatedTo, infofilters, conffilters, metricfilters, eventfilters,
confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
metricsTimeStart, metricsTimeEnd, fromId);
metricsTimeStart, metricsTimeEnd, fromId, false);
}
/**
@ -2310,7 +2342,7 @@ public Set<TimelineEntity> getFlowApps(
null, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo,
infofilters, conffilters, metricfilters, eventfilters,
confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
metricsTimeStart, metricsTimeEnd, fromId);
metricsTimeStart, metricsTimeEnd, fromId, false);
}
/**
@ -2420,7 +2452,7 @@ public Set<TimelineEntity> getFlowApps(
null, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo,
infofilters, conffilters, metricfilters, eventfilters,
confsToRetrieve, metricsToRetrieve, fields, metricsLimit,
metricsTimeStart, metricsTimeEnd, fromId);
metricsTimeStart, metricsTimeEnd, fromId, false);
}
/**