big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-6280. Introduce deselect query param to skip ResourceRequest from getApp/getApps REST API. Contributed by Lantao Jin.

Browse Source
This commit is contained in:
Sunil G 2017-06-28 15:40:58 -07:00
parent 4e3eebc943
commit c1edca101c
7 changed files with 408 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

127
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java Normal file
View File

@ -0,0 +1,127 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements.See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership.The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License.You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.webapp.BadRequestException;
/**
* DeSelectFields make the <code>/apps</code> api more flexible.
* It can be used to strip off more fields if there's such use case in future.
* You can simply extend it via two steps:
* <br> 1. add a <code>DeSelectType</code> enum with a string literals
* <br> 2. write your logical based on
* the return of method contains(DeSelectType)
*/
public class DeSelectFields {
private static final Log LOG =
LogFactory.getLog(DeSelectFields.class.getName());
private final Set<DeSelectType> types;
public DeSelectFields() {
this.types = new HashSet<DeSelectType>();
}
/**
* Initial DeSelectFields with unselected fields.
* @param unselectedFields a set of unselected field.
*/
public void initFields(Set<String> unselectedFields) {
if (unselectedFields == null) {
return;
}
for (String field : unselectedFields) {
if (!field.trim().isEmpty()) {
String[] literalsArray = field.split(",");
for (String literals : literalsArray) {
if (literals != null && !literals.trim().isEmpty()) {
DeSelectType type = DeSelectType.obtainType(literals);
if (type == null) {
LOG.warn("Invalid deSelects string " + literals.trim());
DeSelectType[] typeArray = DeSelectType.values();
String allSuppportLiterals = Arrays.toString(typeArray);
throw new BadRequestException("Invalid deSelects string "
+ literals.trim() + " specified. It should be one of "
+ allSuppportLiterals);
} else {
this.types.add(type);
}
}
}
}
}
}
/**
* Determine the deselect type should be handled or not.
* @param type deselected type
* @return true if the deselect type should be handled
*/
public boolean contains(DeSelectType type) {
return types.contains(type);
}
/**
* Deselect field type, can be boost in future.
*/
public enum DeSelectType {
/**
* <code>RESOURCE_REQUESTS</code> is the first
* supported type from YARN-6280.
*/
RESOURCE_REQUESTS("resourceRequests");
private final String literals;
DeSelectType(String literals) {
this.literals = literals;
}
/**
* use literals as toString.
* @return the literals of this type.
*/
@Override
public String toString() {
return literals;
}
/**
* Obtain the <code>DeSelectType</code> by the literals given behind
* <code>deSelects</code> in URL.
* <br> e.g: deSelects="resourceRequests"
* @param literals e.g: resourceRequests
* @return <code>DeSelectType</code> e.g: DeSelectType.RESOURCE_REQUESTS
*/
public static DeSelectType obtainType(String literals) {
for (DeSelectType type : values()) {
if (type.literals.equalsIgnoreCase(literals)) {
return type;
}
}
return null;
}
}
}

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServiceProtocol.java
View File

@ -154,6 +154,7 @@ String dumpSchedulerLogs(String time, HttpServletRequest hsr)
* @param finishEnd filter the result by finish end time
* @param applicationTypes filter the result by types
* @param applicationTags filter the result by tags
* @param unselectedFields De-selected params to avoid from report
* @return all apps in the cluster
*/
@SuppressWarnings("checkstyle:parameternumber")
@ -161,7 +162,7 @@ AppsInfo getApps(HttpServletRequest hsr, String stateQuery,
Set<String> statesQuery, String finalStatusQuery, String userQuery,
String queueQuery, String count, String startedBegin, String startedEnd,
String finishBegin, String finishEnd, Set<String> applicationTypes,
Set<String> applicationTags);
Set<String> applicationTags, Set<String> unselectedFields);
/**
* This method retrieve all the activities in a specific node, and it is
@ -205,9 +206,11 @@ ApplicationStatisticsInfo getAppStatistics(HttpServletRequest hsr,
* @see ApplicationClientProtocol#getApplicationReport
* @param hsr the servlet request
* @param appId the Id of the application we want the report
* @param unselectedFields De-selected param list to avoid from report
* @return the app report for a specific application
*/
AppInfo getApp(HttpServletRequest hsr, String appId);
AppInfo getApp(HttpServletRequest hsr, String appId,
Set<String> unselectedFields);
/**
* This method retrieves the state for a specific app, and it is reachable by

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
View File

@ -444,7 +444,8 @@ public AppsInfo getApps(@Context HttpServletRequest hsr,
@QueryParam(RMWSConsts.FINISHED_TIME_BEGIN) String finishBegin,
@QueryParam(RMWSConsts.FINISHED_TIME_END) String finishEnd,
@QueryParam(RMWSConsts.APPLICATION_TYPES) Set<String> applicationTypes,
@QueryParam(RMWSConsts.APPLICATION_TAGS) Set<String> applicationTags) {
@QueryParam(RMWSConsts.APPLICATION_TAGS) Set<String> applicationTags,
@QueryParam("deSelects") Set<String> unselectedFields) {
boolean checkCount = false;
boolean checkStart = false;
boolean checkEnd = false;
@ -601,8 +602,11 @@ public AppsInfo getApps(@Context HttpServletRequest hsr,
}
}
DeSelectFields deSelectFields = new DeSelectFields();
deSelectFields.initFields(unselectedFields);
AppInfo app = new AppInfo(rm, rmapp, hasAccess(rmapp, hsr),
WebAppUtils.getHttpSchemePrefix(conf));
WebAppUtils.getHttpSchemePrefix(conf), deSelectFields);
allApps.add(app);
}
return allApps;
@ -827,14 +831,20 @@ private static void countApp(
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
@Override
public AppInfo getApp(@Context HttpServletRequest hsr,
@PathParam(RMWSConsts.APPID) String appId) {
@PathParam(RMWSConsts.APPID) String appId,
@QueryParam("deSelects") Set<String> unselectedFields) {
init();
ApplicationId id = WebAppUtils.parseApplicationId(recordFactory, appId);
RMApp app = rm.getRMContext().getRMApps().get(id);
if (app == null) {
throw new NotFoundException("app with id: " + appId + " not found");
}
return new AppInfo(rm, app, hasAccess(app, hsr), hsr.getScheme() + "://");
DeSelectFields deSelectFields = new DeSelectFields();
deSelectFields.initFields(unselectedFields);
return new AppInfo(rm, app, hasAccess(app, hsr), hsr.getScheme() + "://",
deSelectFields);
}
@GET

27
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
View File

@ -44,6 +44,8 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.DeSelectFields;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.DeSelectFields.DeSelectType;
import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
@ -121,9 +123,14 @@ public class AppInfo {
public AppInfo() {
} // JAXB needs this
@SuppressWarnings({ "rawtypes", "unchecked" })
public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
String schemePrefix) {
this(rm, app, hasAccess, schemePrefix, new DeSelectFields());
}
@SuppressWarnings({ "rawtypes", "unchecked" })
public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
String schemePrefix, DeSelectFields deSelects) {
this.schemePrefix = schemePrefix;
if (app != null) {
String trackingUrl = app.getTrackingUrl();
@ -196,13 +203,19 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
clusterUsagePercentage = resourceReport.getClusterUsagePercentage();
}
List<ResourceRequest> resourceRequestsRaw = rm.getRMContext()
.getScheduler()
.getPendingResourceRequestsForAttempt(attempt.getAppAttemptId());
/* When the deSelects parameter contains "resourceRequests",
it skips returning massive ResourceRequest objects and vice versa.
Default behavior is no skipping. (YARN-6280)
*/
if (!deSelects.contains(DeSelectType.RESOURCE_REQUESTS)) {
List<ResourceRequest> resourceRequestsRaw = rm.getRMContext()
.getScheduler().getPendingResourceRequestsForAttempt(
attempt.getAppAttemptId());
if (resourceRequestsRaw != null) {
for (ResourceRequest req : resourceRequestsRaw) {
resourceRequests.add(new ResourceRequestInfo(req));
if (resourceRequestsRaw != null) {
for (ResourceRequest req : resourceRequestsRaw) {
resourceRequests.add(new ResourceRequestInfo(req));
}
}
}
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
View File

@ -665,12 +665,12 @@ public void testAppsRace() throws Exception {
// verify we don't get any apps when querying
HttpServletRequest mockHsr = mock(HttpServletRequest.class);
AppsInfo appsInfo = webSvc.getApps(mockHsr, null, emptySet, null,
null, null, null, null, null, null, null, emptySet, emptySet);
null, null, null, null, null, null, null, emptySet, emptySet, null);
assertTrue(appsInfo.getApps().isEmpty());
// verify we don't get an NPE when specifying a final status query
appsInfo = webSvc.getApps(mockHsr, null, emptySet, "FAILED",
null, null, null, null, null, null, null, emptySet, emptySet);
null, null, null, null, null, null, null, emptySet, emptySet, null);
assertTrue(appsInfo.getApps().isEmpty());
}

64
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
View File

@ -1054,6 +1054,70 @@ public void testAppsQueryAppTypes() throws JSONException, Exception {
rm.stop();
}
@Test
public void testAppsQueryWithInvaildDeselects()
throws JSONException, Exception {
try {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
rm.submitApp(CONTAINER_MB);
amNodeManager.nodeHeartbeat(true);
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("apps").queryParam("deSelects", "INVALIED_deSelectsParam")
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertResponseStatusCode(Status.BAD_REQUEST, response.getStatusInfo());
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
response.getType().toString());
JSONObject msg = response.getEntity(JSONObject.class);
JSONObject exception = msg.getJSONObject("RemoteException");
assertEquals("incorrect number of elements", 3, exception.length());
String message = exception.getString("message");
String type = exception.getString("exception");
String classname = exception.getString("javaClassName");
WebServicesTestUtils.checkStringContains("exception message",
"java.lang.Exception: Invalid deSelects string"
+ " INVALIED_deSelectsParam " + "specified. It should be one of",
message);
WebServicesTestUtils.checkStringEqual("exception type",
"BadRequestException", type);
WebServicesTestUtils.checkStringEqual("exception classname",
"org.apache.hadoop.yarn.webapp.BadRequestException", classname);
} finally {
rm.stop();
}
}
@Test
public void testAppsQueryWithDeselects()
throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
rm.submitApp(CONTAINER_MB);
amNodeManager.nodeHeartbeat(true);
WebResource r = resource();
MultivaluedMapImpl params = new MultivaluedMapImpl();
params.add("deSelects",
DeSelectFields.DeSelectType.RESOURCE_REQUESTS.toString());
ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("apps").queryParams(params)
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
response.getType().toString());
JSONObject json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
JSONObject apps = json.getJSONObject("apps");
assertEquals("incorrect number of elements", 1, apps.length());
JSONArray array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", 1, array.length());
JSONObject app = array.getJSONObject(0);
assertTrue("resource requests shouldn't exits",
!app.has("resourceRequests"));
rm.stop();
}
@Test
public void testAppStatistics() throws JSONException, Exception {
try {

178
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
View File

@ -1330,6 +1330,7 @@ Multiple parameters can be specified for GET operations. The started and finishe
* finishedTimeEnd - applications with finish time ending with this time, specified in ms since epoch
* applicationTypes - applications matching the given application types, specified as a comma-separated list.
* applicationTags - applications matching any of the given application tags, specified as a comma-separated list.
* deSelects - a generic fields which will be skipped in the result.
### Elements of the *apps* (Applications) object
@ -1339,6 +1340,21 @@ When you make a request for the list of applications, the information will be re
|:---- |:---- |:---- |
| app | array of app objects(JSON)/zero or more application objects(XML) | The collection of application objects |
###Elements of the *deSelects* parameter
Help requesters who don't need certain information to reduce the overhead.
Current supported items:
| Item | Data Type | Description |
|:---- |:---- |:---- |
| resouceRequests | comma separated string | Skip resource requests of application in return |
e.g:
* http://<rm http address:port>/ws/v1/cluster/apps?deSelects=resouceRequests
### Response Examples
**JSON response**
@ -1396,7 +1412,47 @@ Response Body:
"logAggregationStatus": "DISABLED",
"unmanagedApplication": false,
"appNodeLabelExpression": "",
"amNodeLabelExpression": ""
"amNodeLabelExpression": "",
"resourceRequests": [
{
"capability": {
"memory": 4096,
"virtualCores": 1
},
"nodeLabelExpression": "",
"numContainers": 0,
"priority": {
"priority": 0
},
"relaxLocality": true,
"resourceName": "*"
},
{
"capability": {
"memory": 4096,
"virtualCores": 1
},
"nodeLabelExpression": "",
"numContainers": 0,
"priority": {
"priority": 20
},
"relaxLocality": true,
"resourceName": "host1.domain.com"
},
{
"capability": {
"memory": 4096,
"virtualCores": 1
},
"nodeLabelExpression": "",
"numContainers": 0,
"priority": {
"priority": 20
},
"relaxLocality": true,
"resourceName": "host2.domain.com"
}]
},
{
"id": "application_1476912658570_0001",
@ -1432,7 +1488,47 @@ Response Body:
"logAggregationStatus": "DISABLED",
"unmanagedApplication": false,
"appNodeLabelExpression": "",
"amNodeLabelExpression": ""
"amNodeLabelExpression": "",
"resourceRequests": [
{
"capability": {
"memory": 4096,
"virtualCores": 1
},
"nodeLabelExpression": "",
"numContainers": 0,
"priority": {
"priority": 0
},
"relaxLocality": true,
"resourceName": "*"
},
{
"capability": {
"memory": 4096,
"virtualCores": 1
},
"nodeLabelExpression": "",
"numContainers": 0,
"priority": {
"priority": 20
},
"relaxLocality": true,
"resourceName": "host3.domain.com"
},
{
"capability": {
"memory": 4096,
"virtualCores": 1
},
"nodeLabelExpression": "",
"numContainers": 0,
"priority": {
"priority": 20
},
"relaxLocality": true,
"resourceName": "host4.domain.com"
}]
}
]
}
@ -1493,6 +1589,45 @@ Response Body:
<unmanagedApplication>false</unmanagedApplication>
<appNodeLabelExpression></appNodeLabelExpression>
<amNodeLabelExpression></amNodeLabelExpression>
<resourceRequests>
<capability>
<memory>4096</memory>
<virtualCores>1</virtualCores>
</capability>
<nodeLabelExpression/>
<numContainers>0</numContainers>
<priority>
<priority>0</priority>
</priority>
<relaxLocality>true</relaxLocality>
<resourceName>*</resourceName>
</resourceRequests>
<resourceRequests>
<capability>
<memory>4096</memory>
<virtualCores>1</virtualCores>
</capability>
<nodeLabelExpression/>
<numContainers>0</numContainers>
<priority>
<priority>20</priority>
</priority>
<relaxLocality>true</relaxLocality>
<resourceName>host1.domain.com</resourceName>
</resourceRequests>
<resourceRequests>
<capability>
<memory>4096</memory>
<virtualCores>1</virtualCores>
</capability>
<nodeLabelExpression/>
<numContainers>0</numContainers>
<priority>
<priority>20</priority>
</priority>
<relaxLocality>true</relaxLocality>
<resourceName>host2.domain.com</resourceName>
</resourceRequests>
</app>
<app>
<id>application_1476912658570_0001</id>
@ -1529,6 +1664,45 @@ Response Body:
<unmanagedApplication>false</unmanagedApplication>
<appNodeLabelExpression></appNodeLabelExpression>
<amNodeLabelExpression></amNodeLabelExpression>
<resourceRequests>
<capability>
<memory>4096</memory>
<virtualCores>1</virtualCores>
</capability>
<nodeLabelExpression/>
<numContainers>0</numContainers>
<priority>
<priority>0</priority>
</priority>
<relaxLocality>true</relaxLocality>
<resourceName>*</resourceName>
</resourceRequests>
<resourceRequests>
<capability>
<memory>4096</memory>
<virtualCores>1</virtualCores>
</capability>
<nodeLabelExpression/>
<numContainers>0</numContainers>
<priority>
<priority>20</priority>
</priority>
<relaxLocality>true</relaxLocality>
<resourceName>host1.domain.com</resourceName>
</resourceRequests>
<resourceRequests>
<capability>
<memory>4096</memory>
<virtualCores>1</virtualCores>
</capability>
<nodeLabelExpression/>
<numContainers>0</numContainers>
<priority>
<priority>20</priority>
</priority>
<relaxLocality>true</relaxLocality>
<resourceName>host2.domain.com</resourceName>
</resourceRequests>
</app>
</apps>
```