YARN-6280. Introduce deselect query param to skip ResourceRequest from getApp/getApps REST API. Contributed by Lantao Jin.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java

@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.webapp.BadRequestException;
+
+/**
+ * DeSelectFields make the <code>/apps</code> api more flexible.
+ * It can be used to strip off more fields if there's such use case in future.
+ * You can simply extend it via two steps:
+ * <br> 1. add a <code>DeSelectType</code> enum with a string literals
+ * <br> 2. write your logical based on
+ * the return of method contains(DeSelectType)
+ */
+public class DeSelectFields {
+  private static final Log LOG =
+      LogFactory.getLog(DeSelectFields.class.getName());
+
+  private final Set<DeSelectType> types;
+
+  public DeSelectFields() {
+    this.types = new HashSet<DeSelectType>();
+  }
+
+  /**
+   * Initial DeSelectFields with unselected fields.
+   * @param unselectedFields a set of unselected field.
+   */
+  public void initFields(Set<String> unselectedFields) {
+    if (unselectedFields == null) {
+      return;
+    }
+    for (String field : unselectedFields) {
+      if (!field.trim().isEmpty()) {
+        String[] literalsArray = field.split(",");
+        for (String literals : literalsArray) {
+          if (literals != null && !literals.trim().isEmpty()) {
+            DeSelectType type = DeSelectType.obtainType(literals);
+            if (type == null) {
+              LOG.warn("Invalid deSelects string " + literals.trim());
+              DeSelectType[] typeArray = DeSelectType.values();
+              String allSuppportLiterals = Arrays.toString(typeArray);
+              throw new BadRequestException("Invalid deSelects string "
+                  + literals.trim() + " specified. It should be one of "
+                  + allSuppportLiterals);
+            } else {
+              this.types.add(type);
+            }
+          }
+        }
+      }
+    }
+  }
+
+  /**
+   * Determine the deselect type should be handled or not.
+   * @param type deselected type
+   * @return true if the deselect type should be handled
+   */
+  public boolean contains(DeSelectType type) {
+    return types.contains(type);
+  }
+
+  /**
+   * Deselect field type, can be boost in future.
+   */
+  public enum DeSelectType {
+
+    /**
+     * <code>RESOURCE_REQUESTS</code> is the first
+     * supported type from YARN-6280.
+     */
+    RESOURCE_REQUESTS("resourceRequests");
+
+    private final String literals;
+
+    DeSelectType(String literals) {
+      this.literals = literals;
+    }
+
+    /**
+     * use literals as toString.
+     * @return the literals of this type.
+     */
+    @Override
+    public String toString() {
+      return literals;
+    }
+
+    /**
+     * Obtain the <code>DeSelectType</code> by the literals given behind
+     * <code>deSelects</code> in URL.
+     * <br> e.g: deSelects="resourceRequests"
+     * @param literals e.g: resourceRequests
+     * @return <code>DeSelectType</code> e.g: DeSelectType.RESOURCE_REQUESTS
+     */
+    public static DeSelectType obtainType(String literals) {
+      for (DeSelectType type : values()) {
+        if (type.literals.equalsIgnoreCase(literals)) {
+          return type;
+        }
+      }
+      return null;
+    }
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java

@@ -154,6 +154,7 @@ String dumpSchedulerLogs(String time, HttpServletRequest hsr)
    * @param finishEnd filter the result by finish end time
    * @param applicationTypes filter the result by types
    * @param applicationTags filter the result by tags
+   * @param unselectedFields De-selected params to avoid from report
    * @return all apps in the cluster
    */
   @SuppressWarnings("checkstyle:parameternumber")
@@ -161,7 +162,7 @@ AppsInfo getApps(HttpServletRequest hsr, String stateQuery,
       Set<String> statesQuery, String finalStatusQuery, String userQuery,
       String queueQuery, String count, String startedBegin, String startedEnd,
       String finishBegin, String finishEnd, Set<String> applicationTypes,
-      Set<String> applicationTags);
+      Set<String> applicationTags, Set<String> unselectedFields);
 
   /**
    * This method retrieve all the activities in a specific node, and it is
@@ -205,9 +206,11 @@ ApplicationStatisticsInfo getAppStatistics(HttpServletRequest hsr,
    * @see ApplicationClientProtocol#getApplicationReport
    * @param hsr the servlet request
    * @param appId the Id of the application we want the report
+   * @param unselectedFields De-selected param list to avoid from report
    * @return the app report for a specific application
    */
-  AppInfo getApp(HttpServletRequest hsr, String appId);
+  AppInfo getApp(HttpServletRequest hsr, String appId,
+      Set<String> unselectedFields);
 
   /**
    * This method retrieves the state for a specific app, and it is reachable by

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java

@@ -444,7 +444,8 @@ public AppsInfo getApps(@Context HttpServletRequest hsr,
       @QueryParam(RMWSConsts.FINISHED_TIME_BEGIN) String finishBegin,
       @QueryParam(RMWSConsts.FINISHED_TIME_END) String finishEnd,
       @QueryParam(RMWSConsts.APPLICATION_TYPES) Set<String> applicationTypes,
-      @QueryParam(RMWSConsts.APPLICATION_TAGS) Set<String> applicationTags) {
+      @QueryParam(RMWSConsts.APPLICATION_TAGS) Set<String> applicationTags,
+      @QueryParam("deSelects") Set<String> unselectedFields) {
     boolean checkCount = false;
     boolean checkStart = false;
     boolean checkEnd = false;
@@ -601,8 +602,11 @@ public AppsInfo getApps(@Context HttpServletRequest hsr,
         }
       }
 
+      DeSelectFields deSelectFields = new DeSelectFields();
+      deSelectFields.initFields(unselectedFields);
+
       AppInfo app = new AppInfo(rm, rmapp, hasAccess(rmapp, hsr),
-          WebAppUtils.getHttpSchemePrefix(conf));
+          WebAppUtils.getHttpSchemePrefix(conf), deSelectFields);
       allApps.add(app);
     }
     return allApps;
@@ -827,14 +831,20 @@ private static void countApp(
       MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
   @Override
   public AppInfo getApp(@Context HttpServletRequest hsr,
-      @PathParam(RMWSConsts.APPID) String appId) {
+      @PathParam(RMWSConsts.APPID) String appId,
+      @QueryParam("deSelects") Set<String> unselectedFields) {
     init();
     ApplicationId id = WebAppUtils.parseApplicationId(recordFactory, appId);
     RMApp app = rm.getRMContext().getRMApps().get(id);
     if (app == null) {
       throw new NotFoundException("app with id: " + appId + " not found");
     }
-    return new AppInfo(rm, app, hasAccess(app, hsr), hsr.getScheme() + "://");
+
+    DeSelectFields deSelectFields = new DeSelectFields();
+    deSelectFields.initFields(unselectedFields);
+
+    return new AppInfo(rm, app, hasAccess(app, hsr), hsr.getScheme() + "://",
+        deSelectFields);
   }
 
   @GET

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java

@@ -44,6 +44,8 @@
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.DeSelectFields;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.DeSelectFields.DeSelectType;
 import org.apache.hadoop.yarn.util.Times;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 
@@ -121,9 +123,14 @@ public class AppInfo {
   public AppInfo() {
   } // JAXB needs this
 
-  @SuppressWarnings({ "rawtypes", "unchecked" })
   public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
       String schemePrefix) {
+    this(rm, app, hasAccess, schemePrefix, new DeSelectFields());
+  }
+
+  @SuppressWarnings({ "rawtypes", "unchecked" })
+  public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
+      String schemePrefix, DeSelectFields deSelects) {
     this.schemePrefix = schemePrefix;
     if (app != null) {
       String trackingUrl = app.getTrackingUrl();
@@ -196,13 +203,19 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
             clusterUsagePercentage = resourceReport.getClusterUsagePercentage();
           }
 
-          List<ResourceRequest> resourceRequestsRaw = rm.getRMContext()
+          /* When the deSelects parameter contains "resourceRequests",
-              .getScheduler()
+             it skips returning massive ResourceRequest objects and vice versa.
-              .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId());
+             Default behavior is no skipping. (YARN-6280)
+          */
+          if (!deSelects.contains(DeSelectType.RESOURCE_REQUESTS)) {
+            List<ResourceRequest> resourceRequestsRaw = rm.getRMContext()
+                .getScheduler().getPendingResourceRequestsForAttempt(
+                    attempt.getAppAttemptId());
 
-          if (resourceRequestsRaw != null) {
+            if (resourceRequestsRaw != null) {
-            for (ResourceRequest req : resourceRequestsRaw) {
+              for (ResourceRequest req : resourceRequestsRaw) {
-              resourceRequests.add(new ResourceRequestInfo(req));
+                resourceRequests.add(new ResourceRequestInfo(req));
+              }
             }
           }
         }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java

@@ -665,12 +665,12 @@ public void testAppsRace() throws Exception {
     // verify we don't get any apps when querying
     HttpServletRequest mockHsr = mock(HttpServletRequest.class);
     AppsInfo appsInfo = webSvc.getApps(mockHsr, null, emptySet, null,
-        null, null, null, null, null, null, null, emptySet, emptySet);
+        null, null, null, null, null, null, null, emptySet, emptySet, null);
     assertTrue(appsInfo.getApps().isEmpty());
 
     // verify we don't get an NPE when specifying a final status query
     appsInfo = webSvc.getApps(mockHsr, null, emptySet, "FAILED",
-        null, null, null, null, null, null, null, emptySet, emptySet);
+        null, null, null, null, null, null, null, emptySet, emptySet, null);
     assertTrue(appsInfo.getApps().isEmpty());
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java

@@ -1054,6 +1054,70 @@ public void testAppsQueryAppTypes() throws JSONException, Exception {
     rm.stop();
   }
 
+  @Test
+  public void testAppsQueryWithInvaildDeselects()
+      throws JSONException, Exception {
+    try {
+      rm.start();
+      MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
+      rm.submitApp(CONTAINER_MB);
+      amNodeManager.nodeHeartbeat(true);
+      WebResource r = resource();
+      ClientResponse response = r.path("ws").path("v1").path("cluster")
+          .path("apps").queryParam("deSelects", "INVALIED_deSelectsParam")
+          .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+      assertResponseStatusCode(Status.BAD_REQUEST, response.getStatusInfo());
+      assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+          response.getType().toString());
+      JSONObject msg = response.getEntity(JSONObject.class);
+      JSONObject exception = msg.getJSONObject("RemoteException");
+      assertEquals("incorrect number of elements", 3, exception.length());
+      String message = exception.getString("message");
+      String type = exception.getString("exception");
+      String classname = exception.getString("javaClassName");
+      WebServicesTestUtils.checkStringContains("exception message",
+          "java.lang.Exception: Invalid deSelects string"
+              + " INVALIED_deSelectsParam " + "specified. It should be one of",
+          message);
+      WebServicesTestUtils.checkStringEqual("exception type",
+          "BadRequestException", type);
+      WebServicesTestUtils.checkStringEqual("exception classname",
+          "org.apache.hadoop.yarn.webapp.BadRequestException", classname);
+    } finally {
+      rm.stop();
+    }
+  }
+
+  @Test
+  public void testAppsQueryWithDeselects()
+      throws JSONException, Exception {
+    rm.start();
+    MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
+    rm.submitApp(CONTAINER_MB);
+    amNodeManager.nodeHeartbeat(true);
+    WebResource r = resource();
+
+    MultivaluedMapImpl params = new MultivaluedMapImpl();
+    params.add("deSelects",
+        DeSelectFields.DeSelectType.RESOURCE_REQUESTS.toString());
+    ClientResponse response = r.path("ws").path("v1").path("cluster")
+        .path("apps").queryParams(params)
+        .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+    assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+        response.getType().toString());
+
+    JSONObject json = response.getEntity(JSONObject.class);
+    assertEquals("incorrect number of elements", 1, json.length());
+    JSONObject apps = json.getJSONObject("apps");
+    assertEquals("incorrect number of elements", 1, apps.length());
+    JSONArray array = apps.getJSONArray("app");
+    assertEquals("incorrect number of elements", 1, array.length());
+    JSONObject app = array.getJSONObject(0);
+    assertTrue("resource requests shouldn't exits",
+        !app.has("resourceRequests"));
+    rm.stop();
+  }
+
   @Test
   public void testAppStatistics() throws JSONException, Exception {
     try {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md

@@ -1330,6 +1330,7 @@ Multiple parameters can be specified for GET operations. The started and finishe
       * finishedTimeEnd - applications with finish time ending with this time, specified in ms since epoch
       * applicationTypes - applications matching the given application types, specified as a comma-separated list.
       * applicationTags - applications matching any of the given application tags, specified as a comma-separated list.
+      * deSelects - a generic fields which will be skipped in the result.
 
 ### Elements of the *apps* (Applications) object
 
@@ -1339,6 +1340,21 @@ When you make a request for the list of applications, the information will be re
 |:---- |:---- |:---- |
 | app | array of app objects(JSON)/zero or more application objects(XML) | The collection of application objects |
 
+###Elements of the *deSelects* parameter
+
+Help requesters who don't need certain information to reduce the overhead.
+
+Current supported items:
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| resouceRequests | comma separated string | Skip resource requests of application in return |
+
+e.g:
+
+      * http://<rm http address:port>/ws/v1/cluster/apps?deSelects=resouceRequests
+
+
 ### Response Examples
 
 **JSON response**
@@ -1396,7 +1412,47 @@ Response Body:
         "logAggregationStatus": "DISABLED",
         "unmanagedApplication": false,
         "appNodeLabelExpression": "",
-        "amNodeLabelExpression": ""
+        "amNodeLabelExpression": "",
+        "resourceRequests": [
+        {
+            "capability": {
+                "memory": 4096,
+                "virtualCores": 1
+            },
+            "nodeLabelExpression": "",
+            "numContainers": 0,
+            "priority": {
+                "priority": 0
+            },
+            "relaxLocality": true,
+            "resourceName": "*"
+        },
+        {
+            "capability": {
+                "memory": 4096,
+                "virtualCores": 1
+            },
+            "nodeLabelExpression": "",
+            "numContainers": 0,
+            "priority": {
+                "priority": 20
+            },
+            "relaxLocality": true,
+            "resourceName": "host1.domain.com"
+        },
+        {
+            "capability": {
+                "memory": 4096,
+                "virtualCores": 1
+            },
+            "nodeLabelExpression": "",
+            "numContainers": 0,
+            "priority": {
+                "priority": 20
+            },
+            "relaxLocality": true,
+            "resourceName": "host2.domain.com"
+        }]
       },
       {
         "id": "application_1476912658570_0001",
@@ -1432,7 +1488,47 @@ Response Body:
         "logAggregationStatus": "DISABLED",
         "unmanagedApplication": false,
         "appNodeLabelExpression": "",
-        "amNodeLabelExpression": ""
+        "amNodeLabelExpression": "",
+        "resourceRequests": [
+        {
+            "capability": {
+                "memory": 4096,
+                "virtualCores": 1
+            },
+            "nodeLabelExpression": "",
+            "numContainers": 0,
+            "priority": {
+                "priority": 0
+            },
+            "relaxLocality": true,
+            "resourceName": "*"
+        },
+        {
+            "capability": {
+                "memory": 4096,
+                "virtualCores": 1
+            },
+            "nodeLabelExpression": "",
+            "numContainers": 0,
+            "priority": {
+                "priority": 20
+            },
+            "relaxLocality": true,
+            "resourceName": "host3.domain.com"
+        },
+        {
+            "capability": {
+                "memory": 4096,
+                "virtualCores": 1
+            },
+            "nodeLabelExpression": "",
+            "numContainers": 0,
+            "priority": {
+                "priority": 20
+            },
+            "relaxLocality": true,
+            "resourceName": "host4.domain.com"
+        }]
       }
     ]
   }
@@ -1493,6 +1589,45 @@ Response Body:
         <unmanagedApplication>false</unmanagedApplication>
         <appNodeLabelExpression></appNodeLabelExpression>
         <amNodeLabelExpression></amNodeLabelExpression>
+        <resourceRequests>
+          <capability>
+            <memory>4096</memory>
+            <virtualCores>1</virtualCores>
+          </capability>
+          <nodeLabelExpression/>
+          <numContainers>0</numContainers>
+          <priority>
+            <priority>0</priority>
+          </priority>
+          <relaxLocality>true</relaxLocality>
+          <resourceName>*</resourceName>
+        </resourceRequests>
+        <resourceRequests>
+          <capability>
+            <memory>4096</memory>
+            <virtualCores>1</virtualCores>
+          </capability>
+          <nodeLabelExpression/>
+          <numContainers>0</numContainers>
+          <priority>
+            <priority>20</priority>
+          </priority>
+          <relaxLocality>true</relaxLocality>
+          <resourceName>host1.domain.com</resourceName>
+        </resourceRequests>
+        <resourceRequests>
+          <capability>
+            <memory>4096</memory>
+            <virtualCores>1</virtualCores>
+          </capability>
+          <nodeLabelExpression/>
+          <numContainers>0</numContainers>
+          <priority>
+            <priority>20</priority>
+          </priority>
+          <relaxLocality>true</relaxLocality>
+          <resourceName>host2.domain.com</resourceName>
+        </resourceRequests>
     </app>
     <app>
         <id>application_1476912658570_0001</id>
@@ -1529,6 +1664,45 @@ Response Body:
         <unmanagedApplication>false</unmanagedApplication>
         <appNodeLabelExpression></appNodeLabelExpression>
         <amNodeLabelExpression></amNodeLabelExpression>
+        <resourceRequests>
+          <capability>
+            <memory>4096</memory>
+            <virtualCores>1</virtualCores>
+          </capability>
+          <nodeLabelExpression/>
+          <numContainers>0</numContainers>
+          <priority>
+            <priority>0</priority>
+          </priority>
+          <relaxLocality>true</relaxLocality>
+          <resourceName>*</resourceName>
+        </resourceRequests>
+        <resourceRequests>
+          <capability>
+            <memory>4096</memory>
+            <virtualCores>1</virtualCores>
+          </capability>
+          <nodeLabelExpression/>
+          <numContainers>0</numContainers>
+          <priority>
+            <priority>20</priority>
+          </priority>
+          <relaxLocality>true</relaxLocality>
+          <resourceName>host1.domain.com</resourceName>
+        </resourceRequests>
+        <resourceRequests>
+          <capability>
+            <memory>4096</memory>
+            <virtualCores>1</virtualCores>
+          </capability>
+          <nodeLabelExpression/>
+          <numContainers>0</numContainers>
+          <priority>
+            <priority>20</priority>
+          </priority>
+          <relaxLocality>true</relaxLocality>
+          <resourceName>host2.domain.com</resourceName>
+        </resourceRequests>
     </app>
 </apps>
 ```