diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java new file mode 100644 index 0000000000..258bbfa38d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java @@ -0,0 +1,127 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements.See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership.The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License.You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.yarn.webapp.BadRequestException; + +/** + * DeSelectFields make the /apps api more flexible. + * It can be used to strip off more fields if there's such use case in future. + * You can simply extend it via two steps: + *
1. add a DeSelectType enum with a string literals + *
2. write your logical based on + * the return of method contains(DeSelectType) + */ +public class DeSelectFields { + private static final Log LOG = + LogFactory.getLog(DeSelectFields.class.getName()); + + private final Set types; + + public DeSelectFields() { + this.types = new HashSet(); + } + + /** + * Initial DeSelectFields with unselected fields. + * @param unselectedFields a set of unselected field. + */ + public void initFields(Set unselectedFields) { + if (unselectedFields == null) { + return; + } + for (String field : unselectedFields) { + if (!field.trim().isEmpty()) { + String[] literalsArray = field.split(","); + for (String literals : literalsArray) { + if (literals != null && !literals.trim().isEmpty()) { + DeSelectType type = DeSelectType.obtainType(literals); + if (type == null) { + LOG.warn("Invalid deSelects string " + literals.trim()); + DeSelectType[] typeArray = DeSelectType.values(); + String allSuppportLiterals = Arrays.toString(typeArray); + throw new BadRequestException("Invalid deSelects string " + + literals.trim() + " specified. It should be one of " + + allSuppportLiterals); + } else { + this.types.add(type); + } + } + } + } + } + } + + /** + * Determine the deselect type should be handled or not. + * @param type deselected type + * @return true if the deselect type should be handled + */ + public boolean contains(DeSelectType type) { + return types.contains(type); + } + + /** + * Deselect field type, can be boost in future. + */ + public enum DeSelectType { + + /** + * RESOURCE_REQUESTS is the first + * supported type from YARN-6280. + */ + RESOURCE_REQUESTS("resourceRequests"); + + private final String literals; + + DeSelectType(String literals) { + this.literals = literals; + } + + /** + * use literals as toString. + * @return the literals of this type. + */ + @Override + public String toString() { + return literals; + } + + /** + * Obtain the DeSelectType by the literals given behind + * deSelects in URL. + *
e.g: deSelects="resourceRequests" + * @param literals e.g: resourceRequests + * @return DeSelectType e.g: DeSelectType.RESOURCE_REQUESTS + */ + public static DeSelectType obtainType(String literals) { + for (DeSelectType type : values()) { + if (type.literals.equalsIgnoreCase(literals)) { + return type; + } + } + return null; + } + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java index 6dd9c418c8..93ab3de19d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java @@ -154,6 +154,7 @@ String dumpSchedulerLogs(String time, HttpServletRequest hsr) * @param finishEnd filter the result by finish end time * @param applicationTypes filter the result by types * @param applicationTags filter the result by tags + * @param unselectedFields De-selected params to avoid from report * @return all apps in the cluster */ @SuppressWarnings("checkstyle:parameternumber") @@ -161,7 +162,7 @@ AppsInfo getApps(HttpServletRequest hsr, String stateQuery, Set statesQuery, String finalStatusQuery, String userQuery, String queueQuery, String count, String startedBegin, String startedEnd, String finishBegin, String finishEnd, Set applicationTypes, - Set applicationTags); + Set applicationTags, Set unselectedFields); /** * This method retrieve all the activities in a specific node, and it is @@ -205,9 +206,11 @@ ApplicationStatisticsInfo getAppStatistics(HttpServletRequest hsr, * @see ApplicationClientProtocol#getApplicationReport * @param hsr the servlet request * @param appId the Id of the application we want the report + * @param unselectedFields De-selected param list to avoid from report * @return the app report for a specific application */ - AppInfo getApp(HttpServletRequest hsr, String appId); + AppInfo getApp(HttpServletRequest hsr, String appId, + Set unselectedFields); /** * This method retrieves the state for a specific app, and it is reachable by diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index acfb2b89bb..7c053bf388 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -444,7 +444,8 @@ public AppsInfo getApps(@Context HttpServletRequest hsr, @QueryParam(RMWSConsts.FINISHED_TIME_BEGIN) String finishBegin, @QueryParam(RMWSConsts.FINISHED_TIME_END) String finishEnd, @QueryParam(RMWSConsts.APPLICATION_TYPES) Set applicationTypes, - @QueryParam(RMWSConsts.APPLICATION_TAGS) Set applicationTags) { + @QueryParam(RMWSConsts.APPLICATION_TAGS) Set applicationTags, + @QueryParam("deSelects") Set unselectedFields) { boolean checkCount = false; boolean checkStart = false; boolean checkEnd = false; @@ -601,8 +602,11 @@ public AppsInfo getApps(@Context HttpServletRequest hsr, } } + DeSelectFields deSelectFields = new DeSelectFields(); + deSelectFields.initFields(unselectedFields); + AppInfo app = new AppInfo(rm, rmapp, hasAccess(rmapp, hsr), - WebAppUtils.getHttpSchemePrefix(conf)); + WebAppUtils.getHttpSchemePrefix(conf), deSelectFields); allApps.add(app); } return allApps; @@ -827,14 +831,20 @@ private static void countApp( MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) @Override public AppInfo getApp(@Context HttpServletRequest hsr, - @PathParam(RMWSConsts.APPID) String appId) { + @PathParam(RMWSConsts.APPID) String appId, + @QueryParam("deSelects") Set unselectedFields) { init(); ApplicationId id = WebAppUtils.parseApplicationId(recordFactory, appId); RMApp app = rm.getRMContext().getRMApps().get(id); if (app == null) { throw new NotFoundException("app with id: " + appId + " not found"); } - return new AppInfo(rm, app, hasAccess(app, hsr), hsr.getScheme() + "://"); + + DeSelectFields deSelectFields = new DeSelectFields(); + deSelectFields.initFields(unselectedFields); + + return new AppInfo(rm, app, hasAccess(app, hsr), hsr.getScheme() + "://", + deSelectFields); } @GET diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 10e627a441..45eccedc86 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -44,6 +44,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.DeSelectFields; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.DeSelectFields.DeSelectType; import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @@ -121,9 +123,14 @@ public class AppInfo { public AppInfo() { } // JAXB needs this - @SuppressWarnings({ "rawtypes", "unchecked" }) public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, String schemePrefix) { + this(rm, app, hasAccess, schemePrefix, new DeSelectFields()); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, + String schemePrefix, DeSelectFields deSelects) { this.schemePrefix = schemePrefix; if (app != null) { String trackingUrl = app.getTrackingUrl(); @@ -196,13 +203,19 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, clusterUsagePercentage = resourceReport.getClusterUsagePercentage(); } - List resourceRequestsRaw = rm.getRMContext() - .getScheduler() - .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId()); + /* When the deSelects parameter contains "resourceRequests", + it skips returning massive ResourceRequest objects and vice versa. + Default behavior is no skipping. (YARN-6280) + */ + if (!deSelects.contains(DeSelectType.RESOURCE_REQUESTS)) { + List resourceRequestsRaw = rm.getRMContext() + .getScheduler().getPendingResourceRequestsForAttempt( + attempt.getAppAttemptId()); - if (resourceRequestsRaw != null) { - for (ResourceRequest req : resourceRequestsRaw) { - resourceRequests.add(new ResourceRequestInfo(req)); + if (resourceRequestsRaw != null) { + for (ResourceRequest req : resourceRequestsRaw) { + resourceRequests.add(new ResourceRequestInfo(req)); + } } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java index 6af2110d1f..4f7ab54254 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java @@ -665,12 +665,12 @@ public void testAppsRace() throws Exception { // verify we don't get any apps when querying HttpServletRequest mockHsr = mock(HttpServletRequest.class); AppsInfo appsInfo = webSvc.getApps(mockHsr, null, emptySet, null, - null, null, null, null, null, null, null, emptySet, emptySet); + null, null, null, null, null, null, null, emptySet, emptySet, null); assertTrue(appsInfo.getApps().isEmpty()); // verify we don't get an NPE when specifying a final status query appsInfo = webSvc.getApps(mockHsr, null, emptySet, "FAILED", - null, null, null, null, null, null, null, emptySet, emptySet); + null, null, null, null, null, null, null, emptySet, emptySet, null); assertTrue(appsInfo.getApps().isEmpty()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index aab9bee766..056f1dd844 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1054,6 +1054,70 @@ public void testAppsQueryAppTypes() throws JSONException, Exception { rm.stop(); } + @Test + public void testAppsQueryWithInvaildDeselects() + throws JSONException, Exception { + try { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + rm.submitApp(CONTAINER_MB); + amNodeManager.nodeHeartbeat(true); + WebResource r = resource(); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParam("deSelects", "INVALIED_deSelectsParam") + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertResponseStatusCode(Status.BAD_REQUEST, response.getStatusInfo()); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + JSONObject msg = response.getEntity(JSONObject.class); + JSONObject exception = msg.getJSONObject("RemoteException"); + assertEquals("incorrect number of elements", 3, exception.length()); + String message = exception.getString("message"); + String type = exception.getString("exception"); + String classname = exception.getString("javaClassName"); + WebServicesTestUtils.checkStringContains("exception message", + "java.lang.Exception: Invalid deSelects string" + + " INVALIED_deSelectsParam " + "specified. It should be one of", + message); + WebServicesTestUtils.checkStringEqual("exception type", + "BadRequestException", type); + WebServicesTestUtils.checkStringEqual("exception classname", + "org.apache.hadoop.yarn.webapp.BadRequestException", classname); + } finally { + rm.stop(); + } + } + + @Test + public void testAppsQueryWithDeselects() + throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + rm.submitApp(CONTAINER_MB); + amNodeManager.nodeHeartbeat(true); + WebResource r = resource(); + + MultivaluedMapImpl params = new MultivaluedMapImpl(); + params.add("deSelects", + DeSelectFields.DeSelectType.RESOURCE_REQUESTS.toString()); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParams(params) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + JSONArray array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + JSONObject app = array.getJSONObject(0); + assertTrue("resource requests shouldn't exits", + !app.has("resourceRequests")); + rm.stop(); + } + @Test public void testAppStatistics() throws JSONException, Exception { try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md index c119651c74..7c1f418865 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md @@ -1330,6 +1330,7 @@ Multiple parameters can be specified for GET operations. The started and finishe * finishedTimeEnd - applications with finish time ending with this time, specified in ms since epoch * applicationTypes - applications matching the given application types, specified as a comma-separated list. * applicationTags - applications matching any of the given application tags, specified as a comma-separated list. + * deSelects - a generic fields which will be skipped in the result. ### Elements of the *apps* (Applications) object @@ -1339,6 +1340,21 @@ When you make a request for the list of applications, the information will be re |:---- |:---- |:---- | | app | array of app objects(JSON)/zero or more application objects(XML) | The collection of application objects | +###Elements of the *deSelects* parameter + +Help requesters who don't need certain information to reduce the overhead. + +Current supported items: + +| Item | Data Type | Description | +|:---- |:---- |:---- | +| resouceRequests | comma separated string | Skip resource requests of application in return | + +e.g: + + * http:///ws/v1/cluster/apps?deSelects=resouceRequests + + ### Response Examples **JSON response** @@ -1396,7 +1412,47 @@ Response Body: "logAggregationStatus": "DISABLED", "unmanagedApplication": false, "appNodeLabelExpression": "", - "amNodeLabelExpression": "" + "amNodeLabelExpression": "", + "resourceRequests": [ + { + "capability": { + "memory": 4096, + "virtualCores": 1 + }, + "nodeLabelExpression": "", + "numContainers": 0, + "priority": { + "priority": 0 + }, + "relaxLocality": true, + "resourceName": "*" + }, + { + "capability": { + "memory": 4096, + "virtualCores": 1 + }, + "nodeLabelExpression": "", + "numContainers": 0, + "priority": { + "priority": 20 + }, + "relaxLocality": true, + "resourceName": "host1.domain.com" + }, + { + "capability": { + "memory": 4096, + "virtualCores": 1 + }, + "nodeLabelExpression": "", + "numContainers": 0, + "priority": { + "priority": 20 + }, + "relaxLocality": true, + "resourceName": "host2.domain.com" + }] }, { "id": "application_1476912658570_0001", @@ -1432,7 +1488,47 @@ Response Body: "logAggregationStatus": "DISABLED", "unmanagedApplication": false, "appNodeLabelExpression": "", - "amNodeLabelExpression": "" + "amNodeLabelExpression": "", + "resourceRequests": [ + { + "capability": { + "memory": 4096, + "virtualCores": 1 + }, + "nodeLabelExpression": "", + "numContainers": 0, + "priority": { + "priority": 0 + }, + "relaxLocality": true, + "resourceName": "*" + }, + { + "capability": { + "memory": 4096, + "virtualCores": 1 + }, + "nodeLabelExpression": "", + "numContainers": 0, + "priority": { + "priority": 20 + }, + "relaxLocality": true, + "resourceName": "host3.domain.com" + }, + { + "capability": { + "memory": 4096, + "virtualCores": 1 + }, + "nodeLabelExpression": "", + "numContainers": 0, + "priority": { + "priority": 20 + }, + "relaxLocality": true, + "resourceName": "host4.domain.com" + }] } ] } @@ -1493,6 +1589,45 @@ Response Body: false + + + 4096 + 1 + + + 0 + + 0 + + true + * + + + + 4096 + 1 + + + 0 + + 20 + + true + host1.domain.com + + + + 4096 + 1 + + + 0 + + 20 + + true + host2.domain.com + application_1476912658570_0001 @@ -1529,6 +1664,45 @@ Response Body: false + + + 4096 + 1 + + + 0 + + 0 + + true + * + + + + 4096 + 1 + + + 0 + + 20 + + true + host1.domain.com + + + + 4096 + 1 + + + 0 + + 20 + + true + host2.domain.com + ```