diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java index fcab78c78d..db0c4e1fd7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java @@ -2120,4 +2120,743 @@ public Set getFlowApps( infofilters, conffilters, metricfilters, eventfilters, confsToRetrieve, metricsToRetrieve, fields, metricsLimit); } + + /** + * Return a set of application-attempt entities for a given applicationId. + * Cluster ID is not provided by client so default cluster ID has to be taken. + * If userid, flow name and flowrun id which are optional query parameters are + * not specified, they will be queried based on app id and default cluster id + * from the flow context information stored in underlying storage + * implementation. If number of matching entities are more than the limit, + * most recent entities till the limit is reached, will be returned. + * + * @param req Servlet request. + * @param res Servlet response. + * @param appId Application id to which the entities to be queried belong to( + * Mandatory path param). + * @param userId User id which should match for the entities(Optional query + * param) + * @param flowName Flow name which should match for the entities(Optional + * query param). + * @param flowRunId Run id which should match for the entities(Optional query + * param). + * @param limit If specified, defines the number of entities to return. The + * maximum possible value for limit can be {@link Long#MAX_VALUE}. If + * it is not specified or has a value less than 0, then limit will be + * considered as 100. (Optional query param). + * @param createdTimeStart If specified, matched entities should not be + * created before this timestamp(Optional query param). + * @param createdTimeEnd If specified, matched entities should not be created + * after this timestamp(Optional query param). + * @param relatesTo If specified, matched entities should relate to given + * entities associated with a entity type. relatesto is a comma + * separated list in the format + * [entitytype]:[entityid1]:[entityid2]... (Optional query param). + * @param isRelatedTo If specified, matched entities should be related to + * given entities associated with a entity type. relatesto is a comma + * separated list in the format + * [entitytype]:[entityid1]:[entityid2]... (Optional query param). + * @param infofilters If specified, matched entities should have exact matches + * to the given info represented as key-value pairs. This is + * represented as infofilters=info1:value1,info2:value2... (Optional + * query param). + * @param conffilters If specified, matched entities should have exact matches + * to the given configs represented as key-value pairs. This is + * represented as conffilters=conf1:value1,conf2:value2... (Optional + * query param). + * @param metricfilters If specified, matched entities should contain the + * given metrics. This is represented as metricfilters=metricid1, + * metricid2... (Optional query param). + * @param eventfilters If specified, matched entities should contain the given + * events. This is represented as eventfilters=eventid1, eventid2... + * @param confsToRetrieve If specified, defines which configurations to + * retrieve and send back in response. These configs will be + * retrieved irrespective of whether configs are specified in fields + * to retrieve or not. + * @param metricsToRetrieve If specified, defines which metrics to retrieve + * and send back in response. These metrics will be retrieved + * irrespective of whether metrics are specified in fields to + * retrieve or not. + * @param fields Specifies which fields of the entity object to retrieve, see + * {@link Field}. All fields will be retrieved if fields=ALL. If not + * specified, 3 fields i.e. entity type, id, created time is returned + * (Optional query param). + * @param metricsLimit If specified, defines the number of metrics to return. + * Considered only if fields contains METRICS/ALL or + * metricsToRetrieve is specified. Ignored otherwise. The maximum + * possible value for metricsLimit can be {@link Integer#MAX_VALUE}. + * If it is not specified or has a value less than 1, and metrics + * have to be retrieved, then metricsLimit will be considered as 1 + * i.e. latest single value of metric(s) will be returned. (Optional + * query param). + * + * @return If successful, a HTTP 200(OK) response having a JSON representing a + * set of TimelineEntity instances of the app-attempt + * entity type is returned.
+ * On failures,
+ * If any problem occurs in parsing request, HTTP 400(Bad Request) is + * returned.
+ * If flow context information cannot be retrieved, HTTP 404(Not + * Found) is returned.
+ * For all other errors while retrieving data, HTTP 500(Internal + * Server Error) is returned. + */ + @GET + @Path("/apps/{appid}/appattempts") + @Produces(MediaType.APPLICATION_JSON) + public Set getAppAttempts(@Context HttpServletRequest req, + @Context HttpServletResponse res, @PathParam("appid") String appId, + @QueryParam("userid") String userId, + @QueryParam("flowname") String flowName, + @QueryParam("flowrunid") String flowRunId, + @QueryParam("limit") String limit, + @QueryParam("createdtimestart") String createdTimeStart, + @QueryParam("createdtimeend") String createdTimeEnd, + @QueryParam("relatesto") String relatesTo, + @QueryParam("isrelatedto") String isRelatedTo, + @QueryParam("infofilters") String infofilters, + @QueryParam("conffilters") String conffilters, + @QueryParam("metricfilters") String metricfilters, + @QueryParam("eventfilters") String eventfilters, + @QueryParam("confstoretrieve") String confsToRetrieve, + @QueryParam("metricstoretrieve") String metricsToRetrieve, + @QueryParam("fields") String fields, + @QueryParam("metricslimit") String metricsLimit) { + + return getAppAttempts(req, res, null, appId, userId, flowName, flowRunId, + limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo, + infofilters, conffilters, metricfilters, eventfilters, confsToRetrieve, + metricsToRetrieve, fields, metricsLimit); + } + + /** + * Return a set of application-attempt entities for a given applicationId. If + * userid, flow name and flowrun id which are optional query parameters are + * not specified, they will be queried based on app id and cluster id from the + * flow context information stored in underlying storage implementation. If + * number of matching entities are more than the limit, most recent entities + * till the limit is reached, will be returned. + * + * @param req Servlet request. + * @param res Servlet response. + * @param clusterId Cluster id to which the entities to be queried belong to( + * Mandatory path param). + * @param appId Application id to which the entities to be queried belong to( + * Mandatory path param). + * @param userId User id which should match for the entities(Optional query + * param) + * @param flowName Flow name which should match for the entities(Optional + * query param). + * @param flowRunId Run id which should match for the entities(Optional query + * param). + * @param limit If specified, defines the number of entities to return. The + * maximum possible value for limit can be {@link Long#MAX_VALUE}. If + * it is not specified or has a value less than 0, then limit will be + * considered as 100. (Optional query param). + * @param createdTimeStart If specified, matched entities should not be + * created before this timestamp(Optional query param). + * @param createdTimeEnd If specified, matched entities should not be created + * after this timestamp(Optional query param). + * @param relatesTo If specified, matched entities should relate to given + * entities associated with a entity type. relatesto is a comma + * separated list in the format + * [entitytype]:[entityid1]:[entityid2]... (Optional query param). + * @param isRelatedTo If specified, matched entities should be related to + * given entities associated with a entity type. relatesto is a comma + * separated list in the format + * [entitytype]:[entityid1]:[entityid2]... (Optional query param). + * @param infofilters If specified, matched entities should have exact matches + * to the given info represented as key-value pairs. This is + * represented as infofilters=info1:value1,info2:value2... (Optional + * query param). + * @param conffilters If specified, matched entities should have exact matches + * to the given configs represented as key-value pairs. This is + * represented as conffilters=conf1:value1,conf2:value2... (Optional + * query param). + * @param metricfilters If specified, matched entities should contain the + * given metrics. This is represented as metricfilters=metricid1, + * metricid2... (Optional query param). + * @param eventfilters If specified, matched entities should contain the given + * events. This is represented as eventfilters=eventid1, eventid2... + * @param confsToRetrieve If specified, defines which configurations to + * retrieve and send back in response. These configs will be + * retrieved irrespective of whether configs are specified in fields + * to retrieve or not. + * @param metricsToRetrieve If specified, defines which metrics to retrieve + * and send back in response. These metrics will be retrieved + * irrespective of whether metrics are specified in fields to + * retrieve or not. + * @param fields Specifies which fields of the entity object to retrieve, see + * {@link Field}. All fields will be retrieved if fields=ALL. If not + * specified, 3 fields i.e. entity type, id, created time is returned + * (Optional query param). + * @param metricsLimit If specified, defines the number of metrics to return. + * Considered only if fields contains METRICS/ALL or + * metricsToRetrieve is specified. Ignored otherwise. The maximum + * possible value for metricsLimit can be {@link Integer#MAX_VALUE}. + * If it is not specified or has a value less than 1, and metrics + * have to be retrieved, then metricsLimit will be considered as 1 + * i.e. latest single value of metric(s) will be returned. (Optional + * query param). + * + * @return If successful, a HTTP 200(OK) response having a JSON representing a + * set of TimelineEntity instances of the app-attempts + * entity type is returned.
+ * On failures,
+ * If any problem occurs in parsing request, HTTP 400(Bad Request) is + * returned.
+ * If flow context information cannot be retrieved, HTTP 404(Not + * Found) is returned.
+ * For all other errors while retrieving data, HTTP 500(Internal + * Server Error) is returned. + */ + @GET + @Path("/clusters/{clusterid}/apps/{appid}/appattempts") + @Produces(MediaType.APPLICATION_JSON) + public Set getAppAttempts(@Context HttpServletRequest req, + @Context HttpServletResponse res, + @PathParam("clusterid") String clusterId, + @PathParam("appid") String appId, @QueryParam("userid") String userId, + @QueryParam("flowname") String flowName, + @QueryParam("flowrunid") String flowRunId, + @QueryParam("limit") String limit, + @QueryParam("createdtimestart") String createdTimeStart, + @QueryParam("createdtimeend") String createdTimeEnd, + @QueryParam("relatesto") String relatesTo, + @QueryParam("isrelatedto") String isRelatedTo, + @QueryParam("infofilters") String infofilters, + @QueryParam("conffilters") String conffilters, + @QueryParam("metricfilters") String metricfilters, + @QueryParam("eventfilters") String eventfilters, + @QueryParam("confstoretrieve") String confsToRetrieve, + @QueryParam("metricstoretrieve") String metricsToRetrieve, + @QueryParam("fields") String fields, + @QueryParam("metricslimit") String metricsLimit) { + + return getEntities(req, res, clusterId, appId, + TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString(), userId, + flowName, flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo, + isRelatedTo, infofilters, conffilters, metricfilters, eventfilters, + confsToRetrieve, metricsToRetrieve, fields, metricsLimit); + } + + /** + * Return a single application-attempt entity for the given attempt Id. + * Cluster ID is not provided by client so default cluster ID has to be taken. + * If userid, flow name and flowrun id which are optional query parameters are + * not specified, they will be queried based on app id and default cluster id + * from the flow context information stored in underlying storage + * implementation. + * + * @param req Servlet request. + * @param res Servlet response. + * @param appId Application id to which the entity to be queried belongs to( + * Mandatory path param). + * @param appAttemptId Application Attempt Id to which the containers belong + * to( Mandatory path param). + * @param userId User id which should match for the entity(Optional query + * param). + * @param flowName Flow name which should match for the entity(Optional query + * param). + * @param flowRunId Run id which should match for the entity(Optional query + * param). + * @param confsToRetrieve If specified, defines which configurations to + * retrieve and send back in response. These configs will be + * retrieved irrespective of whether configs are specified in fields + * to retrieve or not. + * @param metricsToRetrieve If specified, defines which metrics to retrieve + * and send back in response. These metrics will be retrieved + * irrespective of whether metrics are specified in fields to + * retrieve or not. + * @param fields Specifies which fields of the entity object to retrieve, see + * {@link Field}. All fields will be retrieved if fields=ALL. If not + * specified, 3 fields i.e. entity type, id, created time is returned + * (Optional query param). + * @param metricsLimit If specified, defines the number of metrics to return. + * Considered only if fields contains METRICS/ALL or + * metricsToRetrieve is specified. Ignored otherwise. The maximum + * possible value for metricsLimit can be {@link Integer#MAX_VALUE}. + * If it is not specified or has a value less than 1, and metrics + * have to be retrieved, then metricsLimit will be considered as 1 + * i.e. latest single value of metric(s) will be returned. (Optional + * query param). + * + * @return If successful, a HTTP 200(OK) response having a JSON representing a + * TimelineEntity instance is returned.
+ * On failures,
+ * If any problem occurs in parsing request, HTTP 400(Bad Request) is + * returned.
+ * If flow context information cannot be retrieved or entity for the + * given entity id cannot be found, HTTP 404(Not Found) is + * returned.
+ * For all other errors while retrieving data, HTTP 500(Internal + * Server Error) is returned. + */ + @GET + @Path("/apps/{appid}/appattempts/{appattemptid}") + @Produces(MediaType.APPLICATION_JSON) + public TimelineEntity getAppAttempt(@Context HttpServletRequest req, + @Context HttpServletResponse res, @PathParam("appid") String appId, + @PathParam("appattemptid") String appAttemptId, + @QueryParam("userid") String userId, + @QueryParam("flowname") String flowName, + @QueryParam("flowrunid") String flowRunId, + @QueryParam("confstoretrieve") String confsToRetrieve, + @QueryParam("metricstoretrieve") String metricsToRetrieve, + @QueryParam("fields") String fields, + @QueryParam("metricslimit") String metricsLimit) { + return getAppAttempt(req, res, null, appId, appAttemptId, userId, flowName, + flowRunId, confsToRetrieve, metricsToRetrieve, fields, metricsLimit); + } + + /** + * Return a single application attempt entity of the given entity Id. If + * userid, flowname and flowrun id which are optional query parameters are not + * specified, they will be queried based on app id and cluster id from the + * flow context information stored in underlying storage implementation. + * + * @param req Servlet request. + * @param res Servlet response. + * @param clusterId Cluster id to which the entity to be queried belongs to( + * Mandatory path param). + * @param appId Application id to which the entity to be queried belongs to( + * Mandatory path param). + * @param appAttemptId Application Attempt Id to which the containers belong + * to( Mandatory path param). + * @param userId User id which should match for the entity(Optional query + * param). + * @param flowName Flow name which should match for the entity(Optional query + * param). + * @param flowRunId Run id which should match for the entity(Optional query + * param). + * @param confsToRetrieve If specified, defines which configurations to + * retrieve and send back in response. These configs will be + * retrieved irrespective of whether configs are specified in fields + * to retrieve or not. + * @param metricsToRetrieve If specified, defines which metrics to retrieve + * and send back in response. These metrics will be retrieved + * irrespective of whether metrics are specified in fields to + * retrieve or not. + * @param fields Specifies which fields of the entity object to retrieve, see + * {@link Field}. All fields will be retrieved if fields=ALL. If not + * specified, 3 fields i.e. entity type, id and created time is + * returned (Optional query param). + * @param metricsLimit If specified, defines the number of metrics to return. + * Considered only if fields contains METRICS/ALL or + * metricsToRetrieve is specified. Ignored otherwise. The maximum + * possible value for metricsLimit can be {@link Integer#MAX_VALUE}. + * If it is not specified or has a value less than 1, and metrics + * have to be retrieved, then metricsLimit will be considered as 1 + * i.e. latest single value of metric(s) will be returned. (Optional + * query param). + * + * @return If successful, a HTTP 200(OK) response having a JSON representing a + * TimelineEntity instance is returned.
+ * On failures,
+ * If any problem occurs in parsing request, HTTP 400(Bad Request) is + * returned.
+ * If flow context information cannot be retrieved or entity for the + * given entity id cannot be found, HTTP 404(Not Found) is + * returned.
+ * For all other errors while retrieving data, HTTP 500(Internal + * Server Error) is returned. + */ + @GET + @Path("/clusters/{clusterid}/apps/{appid}/appattempts/{appattemptid}") + @Produces(MediaType.APPLICATION_JSON) + public TimelineEntity getAppAttempt(@Context HttpServletRequest req, + @Context HttpServletResponse res, + @PathParam("clusterid") String clusterId, + @PathParam("appid") String appId, + @PathParam("appattemptid") String appAttemptId, + @QueryParam("userid") String userId, + @QueryParam("flowname") String flowName, + @QueryParam("flowrunid") String flowRunId, + @QueryParam("confstoretrieve") String confsToRetrieve, + @QueryParam("metricstoretrieve") String metricsToRetrieve, + @QueryParam("fields") String fields, + @QueryParam("metricslimit") String metricsLimit) { + return getEntity(req, res, clusterId, appId, + TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString(), appAttemptId, + userId, flowName, flowRunId, confsToRetrieve, metricsToRetrieve, fields, + metricsLimit); + } + + /** + * Return a set of container entities belonging to given application attempt + * id. Cluster ID is not provided by client so default cluster ID has to be + * taken. If userid, flow name and flowrun id which are optional query + * parameters are not specified, they will be queried based on app id and + * default cluster id from the flow context information stored in underlying + * storage implementation. If number of matching entities are more than the + * limit, most recent entities till the limit is reached, will be returned. + * + * @param req Servlet request. + * @param res Servlet response. + * @param appId Application id to which the entities to be queried belong to( + * Mandatory path param). + * @param appattemptId Application Attempt Id to which the containers belong + * to( Mandatory path param). + * @param userId User id which should match for the entities(Optional query + * param) + * @param flowName Flow name which should match for the entities(Optional + * query param). + * @param flowRunId Run id which should match for the entities(Optional query + * param). + * @param limit If specified, defines the number of entities to return. The + * maximum possible value for limit can be {@link Long#MAX_VALUE}. If + * it is not specified or has a value less than 0, then limit will be + * considered as 100. (Optional query param). + * @param createdTimeStart If specified, matched entities should not be + * created before this timestamp(Optional query param). + * @param createdTimeEnd If specified, matched entities should not be created + * after this timestamp(Optional query param). + * @param relatesTo If specified, matched entities should relate to given + * entities associated with a entity type. relatesto is a comma + * separated list in the format + * [entitytype]:[entityid1]:[entityid2]... (Optional query param). + * @param isRelatedTo If specified, matched entities should be related to + * given entities associated with a entity type. relatesto is a comma + * separated list in the format + * [entitytype]:[entityid1]:[entityid2]... (Optional query param). + * @param infofilters If specified, matched entities should have exact matches + * to the given info represented as key-value pairs. This is + * represented as infofilters=info1:value1,info2:value2... (Optional + * query param). + * @param conffilters If specified, matched entities should have exact matches + * to the given configs represented as key-value pairs. This is + * represented as conffilters=conf1:value1,conf2:value2... (Optional + * query param). + * @param metricfilters If specified, matched entities should contain the + * given metrics. This is represented as metricfilters=metricid1, + * metricid2... (Optional query param). + * @param eventfilters If specified, matched entities should contain the given + * events. This is represented as eventfilters=eventid1, eventid2... + * @param confsToRetrieve If specified, defines which configurations to + * retrieve and send back in response. These configs will be + * retrieved irrespective of whether configs are specified in fields + * to retrieve or not. + * @param metricsToRetrieve If specified, defines which metrics to retrieve + * and send back in response. These metrics will be retrieved + * irrespective of whether metrics are specified in fields to + * retrieve or not. + * @param fields Specifies which fields of the entity object to retrieve, see + * {@link Field}. All fields will be retrieved if fields=ALL. If not + * specified, 3 fields i.e. entity type, id, created time is returned + * (Optional query param). + * @param metricsLimit If specified, defines the number of metrics to return. + * Considered only if fields contains METRICS/ALL or + * metricsToRetrieve is specified. Ignored otherwise. The maximum + * possible value for metricsLimit can be {@link Integer#MAX_VALUE}. + * If it is not specified or has a value less than 1, and metrics + * have to be retrieved, then metricsLimit will be considered as 1 + * i.e. latest single value of metric(s) will be returned. (Optional + * query param). + * + * @return If successful, a HTTP 200(OK) response having a JSON representing a + * set of TimelineEntity instances of the containers + * belongs to given application attempt id.
+ * On failures,
+ * If any problem occurs in parsing request, HTTP 400(Bad Request) is + * returned.
+ * If flow context information cannot be retrieved, HTTP 404(Not + * Found) is returned.
+ * For all other errors while retrieving data, HTTP 500(Internal + * Server Error) is returned. + */ + @GET + @Path("/apps/{appid}/appattempts/{appattemptid}/containers") + @Produces(MediaType.APPLICATION_JSON) + public Set getContainers(@Context HttpServletRequest req, + @Context HttpServletResponse res, @PathParam("appid") String appId, + @PathParam("appattemptid") String appattemptId, + @QueryParam("userid") String userId, + @QueryParam("flowname") String flowName, + @QueryParam("flowrunid") String flowRunId, + @QueryParam("limit") String limit, + @QueryParam("createdtimestart") String createdTimeStart, + @QueryParam("createdtimeend") String createdTimeEnd, + @QueryParam("relatesto") String relatesTo, + @QueryParam("isrelatedto") String isRelatedTo, + @QueryParam("infofilters") String infofilters, + @QueryParam("conffilters") String conffilters, + @QueryParam("metricfilters") String metricfilters, + @QueryParam("eventfilters") String eventfilters, + @QueryParam("confstoretrieve") String confsToRetrieve, + @QueryParam("metricstoretrieve") String metricsToRetrieve, + @QueryParam("fields") String fields, + @QueryParam("metricslimit") String metricsLimit) { + return getContainers(req, res, null, appId, appattemptId, userId, flowName, + flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo, + isRelatedTo, infofilters, conffilters, metricfilters, eventfilters, + confsToRetrieve, metricsToRetrieve, fields, metricsLimit); + } + + /** + * Return a set of container entities belonging to given application attempt + * id. If userid, flow name and flowrun id which are optional query parameters + * are not specified, they will be queried based on app id and cluster id from + * the flow context information stored in underlying storage implementation. + * If number of matching entities are more than the limit, most recent + * entities till the limit is reached, will be returned. + * + * @param req Servlet request. + * @param res Servlet response. + * @param clusterId Cluster id to which the entities to be queried belong to( + * Mandatory path param). + * @param appId Application id to which the entities to be queried belong to( + * Mandatory path param). + * @param appattemptId Application Attempt Id to which the containers belong + * to( Mandatory path param). + * @param userId User id which should match for the entities(Optional query + * param) + * @param flowName Flow name which should match for the entities(Optional + * query param). + * @param flowRunId Run id which should match for the entities(Optional query + * param). + * @param limit If specified, defines the number of entities to return. The + * maximum possible value for limit can be {@link Long#MAX_VALUE}. If + * it is not specified or has a value less than 0, then limit will be + * considered as 100. (Optional query param). + * @param createdTimeStart If specified, matched entities should not be + * created before this timestamp(Optional query param). + * @param createdTimeEnd If specified, matched entities should not be created + * after this timestamp(Optional query param). + * @param relatesTo If specified, matched entities should relate to given + * entities associated with a entity type. relatesto is a comma + * separated list in the format + * [entitytype]:[entityid1]:[entityid2]... (Optional query param). + * @param isRelatedTo If specified, matched entities should be related to + * given entities associated with a entity type. relatesto is a comma + * separated list in the format + * [entitytype]:[entityid1]:[entityid2]... (Optional query param). + * @param infofilters If specified, matched entities should have exact matches + * to the given info represented as key-value pairs. This is + * represented as infofilters=info1:value1,info2:value2... (Optional + * query param). + * @param conffilters If specified, matched entities should have exact matches + * to the given configs represented as key-value pairs. This is + * represented as conffilters=conf1:value1,conf2:value2... (Optional + * query param). + * @param metricfilters If specified, matched entities should contain the + * given metrics. This is represented as metricfilters=metricid1, + * metricid2... (Optional query param). + * @param eventfilters If specified, matched entities should contain the given + * events. This is represented as eventfilters=eventid1, eventid2... + * @param confsToRetrieve If specified, defines which configurations to + * retrieve and send back in response. These configs will be + * retrieved irrespective of whether configs are specified in fields + * to retrieve or not. + * @param metricsToRetrieve If specified, defines which metrics to retrieve + * and send back in response. These metrics will be retrieved + * irrespective of whether metrics are specified in fields to + * retrieve or not. + * @param fields Specifies which fields of the entity object to retrieve, see + * {@link Field}. All fields will be retrieved if fields=ALL. If not + * specified, 3 fields i.e. entity type, id, created time is returned + * (Optional query param). + * @param metricsLimit If specified, defines the number of metrics to return. + * Considered only if fields contains METRICS/ALL or + * metricsToRetrieve is specified. Ignored otherwise. The maximum + * possible value for metricsLimit can be {@link Integer#MAX_VALUE}. + * If it is not specified or has a value less than 1, and metrics + * have to be retrieved, then metricsLimit will be considered as 1 + * i.e. latest single value of metric(s) will be returned. (Optional + * query param). + * + * @return If successful, a HTTP 200(OK) response having a JSON representing a + * set of TimelineEntity instances of the containers + * belongs to given application attempt id.
+ * On failures,
+ * If any problem occurs in parsing request, HTTP 400(Bad Request) is + * returned.
+ * If flow context information cannot be retrieved, HTTP 404(Not + * Found) is returned.
+ * For all other errors while retrieving data, HTTP 500(Internal + * Server Error) is returned. + */ + @GET + @Path("/clusters/{clusterid}/apps/{appid}/appattempts/{appattemptid}/containers") + @Produces(MediaType.APPLICATION_JSON) + public Set getContainers(@Context HttpServletRequest req, + @Context HttpServletResponse res, + @PathParam("clusterid") String clusterId, + @PathParam("appid") String appId, + @PathParam("appattemptid") String appattemptId, + @QueryParam("userid") String userId, + @QueryParam("flowname") String flowName, + @QueryParam("flowrunid") String flowRunId, + @QueryParam("limit") String limit, + @QueryParam("createdtimestart") String createdTimeStart, + @QueryParam("createdtimeend") String createdTimeEnd, + @QueryParam("relatesto") String relatesTo, + @QueryParam("isrelatedto") String isRelatedTo, + @QueryParam("infofilters") String infofilters, + @QueryParam("conffilters") String conffilters, + @QueryParam("metricfilters") String metricfilters, + @QueryParam("eventfilters") String eventfilters, + @QueryParam("confstoretrieve") String confsToRetrieve, + @QueryParam("metricstoretrieve") String metricsToRetrieve, + @QueryParam("fields") String fields, + @QueryParam("metricslimit") String metricsLimit) { + + String entityType = TimelineEntityType.YARN_CONTAINER.toString(); + String parentEntityType = + TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString(); + String jsonFormatString = "{\"type\":\"" + parentEntityType + "\",\"id\":\"" + + appattemptId + "\"}"; + String containerFilters = + "SYSTEM_INFO_PARENT_ENTITY eq " + jsonFormatString; + String infofilter; + if (infofilters != null) { + infofilter = containerFilters + " AND " + infofilters; + } else { + infofilter = containerFilters; + } + return getEntities(req, res, clusterId, appId, entityType, userId, flowName, + flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo, + isRelatedTo, infofilter, conffilters, metricfilters, eventfilters, + confsToRetrieve, metricsToRetrieve, fields, metricsLimit); + } + + /** + * Return a single container entity for the given container Id. Cluster ID is + * not provided by client so default cluster ID has to be taken. If userid, + * flow name and flowrun id which are optional query parameters are not + * specified, they will be queried based on app id and default cluster id from + * the flow context information stored in underlying storage implementation. + * + * @param req Servlet request. + * @param res Servlet response. + * @param appId Application id to which the entity to be queried belongs to( + * Mandatory path param). + * @param containerId Container Id to which the entity to be queried belongs + * to( Mandatory path param). + * @param userId User id which should match for the entity(Optional query + * param). + * @param flowName Flow name which should match for the entity(Optional query + * param). + * @param flowRunId Run id which should match for the entity(Optional query + * param). + * @param confsToRetrieve If specified, defines which configurations to + * retrieve and send back in response. These configs will be + * retrieved irrespective of whether configs are specified in fields + * to retrieve or not. + * @param metricsToRetrieve If specified, defines which metrics to retrieve + * and send back in response. These metrics will be retrieved + * irrespective of whether metrics are specified in fields to + * retrieve or not. + * @param fields Specifies which fields of the entity object to retrieve, see + * {@link Field}. All fields will be retrieved if fields=ALL. If not + * specified, 3 fields i.e. entity type, id, created time is returned + * (Optional query param). + * @param metricsLimit If specified, defines the number of metrics to return. + * Considered only if fields contains METRICS/ALL or + * metricsToRetrieve is specified. Ignored otherwise. The maximum + * possible value for metricsLimit can be {@link Integer#MAX_VALUE}. + * If it is not specified or has a value less than 1, and metrics + * have to be retrieved, then metricsLimit will be considered as 1 + * i.e. latest single value of metric(s) will be returned. (Optional + * query param). + * + * @return If successful, a HTTP 200(OK) response having a JSON representing + * TimelineEntity instance is returned.
+ * On failures,
+ * If any problem occurs in parsing request, HTTP 400(Bad Request) is + * returned.
+ * If flow context information cannot be retrieved or entity for the + * given entity id cannot be found, HTTP 404(Not Found) is + * returned.
+ * For all other errors while retrieving data, HTTP 500(Internal + * Server Error) is returned. + */ + @GET + @Path("/apps/{appid}/containers/{containerid}") + @Produces(MediaType.APPLICATION_JSON) + public TimelineEntity getContainer(@Context HttpServletRequest req, + @Context HttpServletResponse res, @PathParam("appid") String appId, + @PathParam("containerid") String containerId, + @QueryParam("userid") String userId, + @QueryParam("flowname") String flowName, + @QueryParam("flowrunid") String flowRunId, + @QueryParam("confstoretrieve") String confsToRetrieve, + @QueryParam("metricstoretrieve") String metricsToRetrieve, + @QueryParam("fields") String fields, + @QueryParam("metricslimit") String metricsLimit) { + return getContainer(req, res, null, appId, containerId, userId, flowName, + flowRunId, confsToRetrieve, metricsToRetrieve, fields, metricsLimit); + } + + /** + * Return a single container entity for the given container Id. If userid, + * flowname and flowrun id which are optional query parameters are not + * specified, they will be queried based on app id and cluster id from the + * flow context information stored in underlying storage implementation. + * + * @param req Servlet request. + * @param res Servlet response. + * @param clusterId Cluster id to which the entity to be queried belongs to( + * Mandatory path param). + * @param appId Application id to which the entity to be queried belongs to( + * Mandatory path param). + * @param containerId Container Id to which the entity to be queried belongs + * to( Mandatory path param). + * @param userId User id which should match for the entity(Optional query + * param). + * @param flowName Flow name which should match for the entity(Optional query + * param). + * @param flowRunId Run id which should match for the entity(Optional query + * param). + * @param confsToRetrieve If specified, defines which configurations to + * retrieve and send back in response. These configs will be + * retrieved irrespective of whether configs are specified in fields + * to retrieve or not. + * @param metricsToRetrieve If specified, defines which metrics to retrieve + * and send back in response. These metrics will be retrieved + * irrespective of whether metrics are specified in fields to + * retrieve or not. + * @param fields Specifies which fields of the entity object to retrieve, see + * {@link Field}. All fields will be retrieved if fields=ALL. If not + * specified, 3 fields i.e. entity type, id and created time is + * returned (Optional query param). + * @param metricsLimit If specified, defines the number of metrics to return. + * Considered only if fields contains METRICS/ALL or + * metricsToRetrieve is specified. Ignored otherwise. The maximum + * possible value for metricsLimit can be {@link Integer#MAX_VALUE}. + * If it is not specified or has a value less than 1, and metrics + * have to be retrieved, then metricsLimit will be considered as 1 + * i.e. latest single value of metric(s) will be returned. (Optional + * query param). + * + * @return If successful, a HTTP 200(OK) response having a JSON representing a + * TimelineEntity instance is returned.
+ * On failures,
+ * If any problem occurs in parsing request, HTTP 400(Bad Request) is + * returned.
+ * If flow context information cannot be retrieved or entity for the + * given entity id cannot be found, HTTP 404(Not Found) is + * returned.
+ * For all other errors while retrieving data, HTTP 500(Internal + * Server Error) is returned. + */ + @GET + @Path("/clusters/{clusterid}/apps/{appid}/containers/{containerid}") + @Produces(MediaType.APPLICATION_JSON) + public TimelineEntity getContainer(@Context HttpServletRequest req, + @Context HttpServletResponse res, + @PathParam("clusterid") String clusterId, + @PathParam("appid") String appId, + @PathParam("containerid") String containerId, + @QueryParam("userid") String userId, + @QueryParam("flowname") String flowName, + @QueryParam("flowrunid") String flowRunId, + @QueryParam("confstoretrieve") String confsToRetrieve, + @QueryParam("metricstoretrieve") String metricsToRetrieve, + @QueryParam("fields") String fields, + @QueryParam("metricslimit") String metricsLimit) { + return getEntity(req, res, clusterId, appId, + TimelineEntityType.YARN_CONTAINER.toString(), containerId, userId, + flowName, flowRunId, confsToRetrieve, metricsToRetrieve, fields, + metricsLimit); + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java index a5ef66cb2a..a2a9c283b6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java @@ -37,6 +37,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineReaderImpl; import org.apache.hadoop.yarn.server.timelineservice.storage.TestFileSystemTimelineReaderImpl; @@ -567,4 +568,188 @@ public void testInvalidValuesHandling() throws Exception { client.destroy(); } } + + @Test + public void testGetAppAttempts() throws Exception { + Client client = createClient(); + try { + URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + "timeline/clusters/cluster1/apps/app1/" + + "entities/YARN_APPLICATION_ATTEMPT"); + ClientResponse resp = getResponse(client, uri); + Set entities = + resp.getEntity(new GenericType>() { + }); + assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType()); + assertNotNull(entities); + int totalEntities = entities.size(); + assertEquals(2, totalEntities); + assertTrue( + "Entity with app-attempt-2 should have been present in response.", + entities.contains( + newEntity(TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString(), + "app-attempt-1"))); + assertTrue( + "Entity with app-attempt-2 should have been present in response.", + entities.contains( + newEntity(TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString(), + "app-attempt-2"))); + + uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + "timeline/clusters/cluster1/apps/app1/appattempts"); + resp = getResponse(client, uri); + entities = resp.getEntity(new GenericType>() { + }); + assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType()); + assertNotNull(entities); + int retrievedEntity = entities.size(); + assertEquals(2, retrievedEntity); + assertTrue( + "Entity with app-attempt-2 should have been present in response.", + entities.contains( + newEntity(TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString(), + "app-attempt-1"))); + assertTrue( + "Entity with app-attempt-2 should have been present in response.", + entities.contains( + newEntity(TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString(), + "app-attempt-2"))); + + assertEquals(totalEntities, retrievedEntity); + + } finally { + client.destroy(); + } + } + + @Test + public void testGetAppAttempt() throws Exception { + Client client = createClient(); + try { + URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + "timeline/clusters/cluster1/apps/app1/entities/" + + "YARN_APPLICATION_ATTEMPT/app-attempt-1"); + ClientResponse resp = getResponse(client, uri); + TimelineEntity entities1 = + resp.getEntity(new GenericType() { + }); + assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType()); + assertNotNull(entities1); + + uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + "timeline/clusters/cluster1/apps/app1/appattempts/app-attempt-1"); + resp = getResponse(client, uri); + TimelineEntity entities2 = + resp.getEntity(new GenericType() { + }); + assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType()); + assertNotNull(entities2); + + assertEquals(entities1, entities2); + + } finally { + client.destroy(); + } + } + + @Test + public void testGetContainers() throws Exception { + Client client = createClient(); + try { + // total 3 containers in a application. + URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + "timeline/clusters/cluster1/apps/app1/entities/YARN_CONTAINER"); + ClientResponse resp = getResponse(client, uri); + Set entities = + resp.getEntity(new GenericType>() { + }); + assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType()); + assertNotNull(entities); + int totalEntities = entities.size(); + assertEquals(3, totalEntities); + assertTrue( + "Entity with container_1_1 should have been present in response.", + entities.contains(newEntity( + TimelineEntityType.YARN_CONTAINER.toString(), "container_1_1"))); + assertTrue( + "Entity with container_2_1 should have been present in response.", + entities.contains(newEntity( + TimelineEntityType.YARN_CONTAINER.toString(), "container_2_1"))); + assertTrue( + "Entity with container_2_2 should have been present in response.", + entities.contains(newEntity( + TimelineEntityType.YARN_CONTAINER.toString(), "container_2_2"))); + + // for app-attempt1 1 container has run + uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + "timeline/clusters/cluster1/apps/app1/" + + "appattempts/app-attempt-1/containers"); + resp = getResponse(client, uri); + entities = resp.getEntity(new GenericType>() { + }); + assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType()); + assertNotNull(entities); + int retrievedEntity = entities.size(); + assertEquals(1, retrievedEntity); + assertTrue( + "Entity with container_1_1 should have been present in response.", + entities.contains(newEntity( + TimelineEntityType.YARN_CONTAINER.toString(), "container_1_1"))); + + // for app-attempt2 2 containers has run + uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + "timeline/clusters/cluster1/apps/app1/" + + "appattempts/app-attempt-2/containers"); + resp = getResponse(client, uri); + entities = resp.getEntity(new GenericType>() { + }); + assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType()); + assertNotNull(entities); + retrievedEntity += entities.size(); + assertEquals(2, entities.size()); + assertTrue( + "Entity with container_2_1 should have been present in response.", + entities.contains(newEntity( + TimelineEntityType.YARN_CONTAINER.toString(), "container_2_1"))); + assertTrue( + "Entity with container_2_2 should have been present in response.", + entities.contains(newEntity( + TimelineEntityType.YARN_CONTAINER.toString(), "container_2_2"))); + + assertEquals(totalEntities, retrievedEntity); + + } finally { + client.destroy(); + } + } + + @Test + public void testGetContainer() throws Exception { + Client client = createClient(); + try { + URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + "timeline/clusters/cluster1/apps/app1/" + + "entities/YARN_CONTAINER/container_2_2"); + ClientResponse resp = getResponse(client, uri); + TimelineEntity entities1 = + resp.getEntity(new GenericType() { + }); + assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType()); + assertNotNull(entities1); + + uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + + "timeline/clusters/cluster1/apps/app1/containers/container_2_2"); + resp = getResponse(client, uri); + TimelineEntity entities2 = + resp.getEntity(new GenericType() { + }); + assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType()); + assertNotNull(entities2); + + assertEquals(entities1, entities2); + + } finally { + client.destroy(); + } + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java index 90f11a50a2..35af16970b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java @@ -34,7 +34,10 @@ import org.apache.commons.csv.CSVPrinter; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationAttemptEntity; +import org.apache.hadoop.yarn.api.records.timelineservice.ContainerEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -118,7 +121,8 @@ private static void writeEntityFile(TimelineEntity entity, File dir) } private static void loadEntityData(String rootDir) throws Exception { - File appDir = getAppDir(rootDir, "cluster1", "user1", "flow1", "1", "app1"); + File appDir = + getAppDir(rootDir, "cluster1", "user1", "flow1", "1", "app1", "app"); TimelineEntity entity11 = new TimelineEntity(); entity11.setId("id_1"); entity11.setType("app"); @@ -259,8 +263,40 @@ private static void loadEntityData(String rootDir) throws Exception { entity4.addEvent(event44); writeEntityFile(entity4, appDir); + File attemptDir = getAppDir(rootDir, "cluster1", "user1", "flow1", "1", + "app1", TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString()); + ApplicationAttemptEntity attempt1 = new ApplicationAttemptEntity(); + attempt1.setId("app-attempt-1"); + attempt1.setCreatedTime(1425017502003L); + writeEntityFile(attempt1, attemptDir); + ApplicationAttemptEntity attempt2 = new ApplicationAttemptEntity(); + attempt2.setId("app-attempt-2"); + attempt2.setCreatedTime(1425017502004L); + writeEntityFile(attempt2, attemptDir); + + File entityDir = getAppDir(rootDir, "cluster1", "user1", "flow1", "1", + "app1", TimelineEntityType.YARN_CONTAINER.toString()); + ContainerEntity containerEntity1 = new ContainerEntity(); + containerEntity1.setId("container_1_1"); + containerEntity1.setParent(attempt1.getIdentifier()); + containerEntity1.setCreatedTime(1425017502003L); + writeEntityFile(containerEntity1, entityDir); + + ContainerEntity containerEntity2 = new ContainerEntity(); + containerEntity2.setId("container_2_1"); + containerEntity2.setParent(attempt2.getIdentifier()); + containerEntity2.setCreatedTime(1425018502003L); + writeEntityFile(containerEntity2, entityDir); + + ContainerEntity containerEntity3 = new ContainerEntity(); + containerEntity3.setId("container_2_2"); + containerEntity3.setParent(attempt2.getIdentifier()); + containerEntity3.setCreatedTime(1425018502003L); + writeEntityFile(containerEntity3, entityDir); + File appDir2 = - getAppDir(rootDir, "cluster1", "user1", "flow1,flow", "1", "app2"); + getAppDir(rootDir, "cluster1", "user1", "flow1,flow", "1", "app2", + "app"); TimelineEntity entity5 = new TimelineEntity(); entity5.setId("id_5"); entity5.setType("app"); @@ -269,11 +305,11 @@ private static void loadEntityData(String rootDir) throws Exception { } private static File getAppDir(String rootDir, String cluster, String user, - String flowName, String flowRunId, String appId) { + String flowName, String flowRunId, String appId, String entityName) { return new File(rootDir + File.separator + "entities" + File.separator + cluster + File.separator + user + File.separator + flowName + File.separator + flowRunId + File.separator + appId + File.separator + - "app" + File.separator); + entityName + File.separator); } @Test