From 8facf1f976d7e12a846f12baabf54be1b7a49f9d Mon Sep 17 00:00:00 2001 From: Sunil G Date: Wed, 27 Sep 2017 14:37:32 +0530 Subject: [PATCH] YARN-6871. Add additional deSelects params in RMWebServices#getAppReport. Contributed by Tanuj Nayak. --- .../webapp/DeSelectFields.java | 11 +- .../resourcemanager/webapp/dao/AppInfo.java | 112 ++++++++++++------ .../webapp/TestRMWebServicesApps.java | 77 +++++++++++- 3 files changed, 162 insertions(+), 38 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java index 258bbfa38d..c991766ae6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java @@ -91,7 +91,16 @@ public enum DeSelectType { * RESOURCE_REQUESTS is the first * supported type from YARN-6280. */ - RESOURCE_REQUESTS("resourceRequests"); + RESOURCE_REQUESTS("resourceRequests"), + /** + * APP_TIMEOUTS, APP_NODE_LABEL_EXPRESSION, AM_NODE_LABEL_EXPRESSION, + * RESOURCE_INFO are additionally supported parameters added in + * YARN-6871. + */ + TIMEOUTS("timeouts"), + APP_NODE_LABEL_EXPRESSION("appNodeLabelExpression"), + AM_NODE_LABEL_EXPRESSION("amNodeLabelExpression"), + RESOURCE_INFO("resourceInfo"); private final String literals; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 236c467765..40d7b4c003 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -123,7 +123,7 @@ public class AppInfo { protected String amNodeLabelExpression; protected ResourcesInfo resourceInfo = null; - protected AppTimeoutsInfo timeouts = new AppTimeoutsInfo(); + private AppTimeoutsInfo timeouts; public AppInfo() { } // JAXB needs this @@ -247,47 +247,87 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, unmanagedApplication = appSubmissionContext.getUnmanagedAM(); appNodeLabelExpression = app.getApplicationSubmissionContext().getNodeLabelExpression(); - amNodeLabelExpression = (unmanagedApplication) ? null - : app.getAMResourceRequests().get(0).getNodeLabelExpression(); + /* + * When the deSelects parameter contains "amNodeLabelExpression", objects + * pertaining to the amNodeLabelExpression are not returned. By default, + * this is not skipped. (YARN-6871) + */ + if(!deSelects.contains(DeSelectType.AM_NODE_LABEL_EXPRESSION)) { + amNodeLabelExpression = (unmanagedApplication) ? + null : + app.getAMResourceRequests().get(0).getNodeLabelExpression(); + } + /* + * When the deSelects parameter contains "appNodeLabelExpression", objects + * pertaining to the appNodeLabelExpression are not returned. By default, + * this is not skipped. (YARN-6871) + */ + if (!deSelects.contains(DeSelectType.APP_NODE_LABEL_EXPRESSION)) { + appNodeLabelExpression = + app.getApplicationSubmissionContext().getNodeLabelExpression(); + } + /* + * When the deSelects parameter contains "amNodeLabelExpression", objects + * pertaining to the amNodeLabelExpression are not returned. By default, + * this is not skipped. (YARN-6871) + */ + if (!deSelects.contains(DeSelectType.AM_NODE_LABEL_EXPRESSION)) { + amNodeLabelExpression = (unmanagedApplication) ? + null : + app.getAMResourceRequests().get(0).getNodeLabelExpression(); + } + /* + * When the deSelects parameter contains "resourceInfo", ResourceInfo + * objects are not returned. Default behavior is no skipping. (YARN-6871) + */ // Setting partition based resource usage of application - ResourceScheduler scheduler = rm.getRMContext().getScheduler(); - if (scheduler instanceof CapacityScheduler) { - RMAppAttempt attempt = app.getCurrentAppAttempt(); - if (null != attempt) { - FiCaSchedulerApp ficaAppAttempt = ((CapacityScheduler) scheduler) - .getApplicationAttempt(attempt.getAppAttemptId()); - resourceInfo = null != ficaAppAttempt - ? new ResourcesInfo(ficaAppAttempt.getSchedulingResourceUsage()) - : null; - } - } - - Map applicationTimeouts = - app.getApplicationTimeouts(); - if (applicationTimeouts.isEmpty()) { - // If application is not set timeout, lifetime should be sent as default - // with expiryTime=UNLIMITED and remainingTime=-1 - AppTimeoutInfo timeoutInfo = new AppTimeoutInfo(); - timeoutInfo.setTimeoutType(ApplicationTimeoutType.LIFETIME); - timeouts.add(timeoutInfo); - } else { - for (Map.Entry entry : app - .getApplicationTimeouts().entrySet()) { - AppTimeoutInfo timeout = new AppTimeoutInfo(); - timeout.setTimeoutType(entry.getKey()); - long timeoutInMillis = entry.getValue().longValue(); - timeout.setExpiryTime(Times.formatISO8601(timeoutInMillis)); - if (app.isAppInCompletedStates()) { - timeout.setRemainingTime(0); - } else { - timeout.setRemainingTime(Math - .max((timeoutInMillis - System.currentTimeMillis()) / 1000, 0)); + if (!deSelects.contains(DeSelectType.RESOURCE_INFO)) { + ResourceScheduler scheduler = rm.getRMContext().getScheduler(); + if (scheduler instanceof CapacityScheduler) { + RMAppAttempt attempt = app.getCurrentAppAttempt(); + if (null != attempt) { + FiCaSchedulerApp ficaAppAttempt = ((CapacityScheduler) scheduler) + .getApplicationAttempt(attempt.getAppAttemptId()); + resourceInfo = null != ficaAppAttempt ? + new ResourcesInfo(ficaAppAttempt.getSchedulingResourceUsage()) : + null; } - timeouts.add(timeout); } } + /* + * When the deSelects parameter contains "appTimeouts", objects pertaining + * to app timeouts are not returned. By default, this is not skipped. + * (YARN-6871) + */ + if (!deSelects.contains(DeSelectType.TIMEOUTS)) { + Map applicationTimeouts = + app.getApplicationTimeouts(); + if (applicationTimeouts.isEmpty()) { + // If application is not set timeout, lifetime should be sent + // as default with expiryTime=UNLIMITED and remainingTime=-1 + AppTimeoutInfo timeoutInfo = new AppTimeoutInfo(); + timeoutInfo.setTimeoutType(ApplicationTimeoutType.LIFETIME); + timeouts = new AppTimeoutsInfo(); + timeouts.add(timeoutInfo); + } else { + for (Map.Entry entry : app + .getApplicationTimeouts().entrySet()) { + AppTimeoutInfo timeout = new AppTimeoutInfo(); + timeout.setTimeoutType(entry.getKey()); + long timeoutInMillis = entry.getValue().longValue(); + timeout.setExpiryTime(Times.formatISO8601(timeoutInMillis)); + if (app.isAppInCompletedStates()) { + timeout.setRemainingTime(0); + } else { + timeout.setRemainingTime(Math.max( + (timeoutInMillis - System.currentTimeMillis()) / 1000, 0)); + } + timeouts.add(timeout); + } + } + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 07f74a358e..f0704ac99e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1113,8 +1113,83 @@ public void testAppsQueryWithDeselects() JSONArray array = apps.getJSONArray("app"); assertEquals("incorrect number of elements", 1, array.length()); JSONObject app = array.getJSONObject(0); - assertTrue("resource requests shouldn't exits", + assertTrue("resource requests shouldn't exist", !app.has("resourceRequests")); + + params.clear(); + params.add("deSelects", + DeSelectFields.DeSelectType.AM_NODE_LABEL_EXPRESSION.toString()); + response = + r.path("ws").path("v1").path("cluster").path("apps").queryParams(params) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + app = array.getJSONObject(0); + assertTrue("AMNodeLabelExpression shouldn't exist", + !app.has("amNodeLabelExpression")); + + params.clear(); + params.add("deSelects", DeSelectFields.DeSelectType.TIMEOUTS.toString()); + response = + r.path("ws").path("v1").path("cluster").path("apps").queryParams(params) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + app = array.getJSONObject(0); + assertTrue("Timeouts shouldn't exist", !app.has("timeouts")); + rm.stop(); + + params.clear(); + params.add("deSelects", + DeSelectFields.DeSelectType.APP_NODE_LABEL_EXPRESSION.toString()); + response = + r.path("ws").path("v1").path("cluster").path("apps").queryParams(params) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + app = array.getJSONObject(0); + assertTrue("AppNodeLabelExpression shouldn't exist", + !app.has("appNodeLabelExpression")); + rm.stop(); + + params.clear(); + params + .add("deSelects", DeSelectFields.DeSelectType.RESOURCE_INFO.toString()); + response = + r.path("ws").path("v1").path("cluster").path("apps").queryParams(params) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + app = array.getJSONObject(0); + assertTrue("Resource info shouldn't exist", !app.has("resourceInfo")); rm.stop(); }