diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 9501969ebf..2560786451 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -89,6 +89,9 @@ Release 2.1.1-beta - UNRELEASED YARN-758. Augment MockNM to use multiple cores (Karthik Kambatla via Sandy Ryza) + YARN-696. Changed RMWebservice apps call to take in multiple application + states. (Trevor Lorimer via vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index a9b1523eea..a5c2f441e0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; import java.io.IOException; +import java.util.Arrays; import java.util.Collection; import java.util.EnumSet; import java.util.HashSet; @@ -231,6 +232,7 @@ public NodeInfo getNode(@PathParam("nodeId") String nodeId) { @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) public AppsInfo getApps(@Context HttpServletRequest hsr, @QueryParam("state") String stateQuery, + @QueryParam("states") Set statesQuery, @QueryParam("finalStatus") String finalStatusQuery, @QueryParam("user") String userQuery, @QueryParam("queue") String queueQuery, @@ -245,6 +247,7 @@ public AppsInfo getApps(@Context HttpServletRequest hsr, boolean checkStart = false; boolean checkEnd = false; boolean checkAppTypes = false; + boolean checkAppStates = false; long countNum = 0; // set values suitable in case both of begin/end not specified @@ -321,6 +324,36 @@ public AppsInfo getApps(@Context HttpServletRequest hsr, checkAppTypes = true; } + String allAppStates; + RMAppState[] stateArray = RMAppState.values(); + allAppStates = Arrays.toString(stateArray); + + Set appStates = new HashSet(); + // stateQuery is deprecated. + if (stateQuery != null && !stateQuery.isEmpty()) { + statesQuery.add(stateQuery); + } + if (!statesQuery.isEmpty()) { + for (String applicationState : statesQuery) { + if (applicationState != null && !applicationState.isEmpty()) { + String[] states = applicationState.split(","); + for (String state : states) { + try { + RMAppState.valueOf(state.trim()); + } catch (IllegalArgumentException iae) { + throw new BadRequestException( + "Invalid application-state " + state + + " specified. It should be one of " + allAppStates); + } + appStates.add(state.trim().toLowerCase()); + } + } + } + } + if (!appStates.isEmpty()) { + checkAppStates = true; + } + final ConcurrentMap apps = rm.getRMContext() .getRMApps(); AppsInfo allApps = new AppsInfo(); @@ -329,11 +362,10 @@ public AppsInfo getApps(@Context HttpServletRequest hsr, if (checkCount && num == countNum) { break; } - if (stateQuery != null && !stateQuery.isEmpty()) { - RMAppState.valueOf(stateQuery); - if (!rmapp.getState().toString().equalsIgnoreCase(stateQuery)) { - continue; - } + + if (checkAppStates + && !appStates.contains(rmapp.getState().toString().toLowerCase())) { + continue; } if (finalStatusQuery != null && !finalStatusQuery.isEmpty()) { FinalApplicationStatus.valueOf(finalStatusQuery); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 52f72d8692..5e7145283c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -68,6 +68,7 @@ import com.sun.jersey.api.client.ClientResponse.Status; import com.sun.jersey.api.client.UniformInterfaceException; import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.core.util.MultivaluedMapImpl; import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; import com.sun.jersey.test.framework.JerseyTest; import com.sun.jersey.test.framework.WebAppDescriptor; @@ -239,6 +240,122 @@ public void testAppsQueryState() throws JSONException, Exception { rm.stop(); } + @Test + public void testAppsQueryStates() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + rm.submitApp(1024); + RMApp killedApp = rm.submitApp(1024); + rm.killApp(killedApp.getApplicationId()); + + amNodeManager.nodeHeartbeat(true); + + WebResource r = resource(); + MultivaluedMapImpl params = new MultivaluedMapImpl(); + params.add("states", RMAppState.ACCEPTED.toString()); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParams(params) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + JSONArray array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + assertEquals("state not equal to ACCEPTED", "ACCEPTED", array + .getJSONObject(0).getString("state")); + + r = resource(); + params = new MultivaluedMapImpl(); + params.add("states", RMAppState.ACCEPTED.toString()); + params.add("states", RMAppState.KILLED.toString()); + response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParams(params) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 2, array.length()); + assertTrue("both app states of ACCEPTED and KILLED are not present", + (array.getJSONObject(0).getString("state").equals("ACCEPTED") && + array.getJSONObject(1).getString("state").equals("KILLED")) || + (array.getJSONObject(0).getString("state").equals("KILLED") && + array.getJSONObject(1).getString("state").equals("ACCEPTED"))); + + rm.stop(); + } + + @Test + public void testAppsQueryStatesComma() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + rm.submitApp(1024); + RMApp killedApp = rm.submitApp(1024); + rm.killApp(killedApp.getApplicationId()); + + amNodeManager.nodeHeartbeat(true); + + WebResource r = resource(); + MultivaluedMapImpl params = new MultivaluedMapImpl(); + params.add("states", RMAppState.ACCEPTED.toString()); + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParams(params) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + JSONArray array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + assertEquals("state not equal to ACCEPTED", "ACCEPTED", array + .getJSONObject(0).getString("state")); + + r = resource(); + params = new MultivaluedMapImpl(); + params.add("states", RMAppState.ACCEPTED.toString() + "," + + RMAppState.KILLED.toString()); + response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParams(params) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 2, array.length()); + assertTrue("both app states of ACCEPTED and KILLED are not present", + (array.getJSONObject(0).getString("state").equals("ACCEPTED") && + array.getJSONObject(1).getString("state").equals("KILLED")) || + (array.getJSONObject(0).getString("state").equals("KILLED") && + array.getJSONObject(1).getString("state").equals("ACCEPTED"))); + + rm.stop(); + } + + @Test + public void testAppsQueryStatesNone() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + rm.submitApp(1024); + amNodeManager.nodeHeartbeat(true); + WebResource r = resource(); + + ClientResponse response = r.path("ws").path("v1").path("cluster") + .path("apps").queryParam("states", RMAppState.RUNNING.toString()) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + assertEquals("apps is not null", JSONObject.NULL, json.get("apps")); + rm.stop(); + } + @Test public void testAppsQueryStateNone() throws JSONException, Exception { rm.start(); @@ -257,6 +374,43 @@ public void testAppsQueryStateNone() throws JSONException, Exception { rm.stop(); } + @Test + public void testAppsQueryStatesInvalid() throws JSONException, Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + rm.submitApp(1024); + amNodeManager.nodeHeartbeat(true); + WebResource r = resource(); + + try { + r.path("ws").path("v1").path("cluster").path("apps") + .queryParam("states", "INVALID_test") + .accept(MediaType.APPLICATION_JSON).get(JSONObject.class); + fail("should have thrown exception on invalid state query"); + } catch (UniformInterfaceException ue) { + ClientResponse response = ue.getResponse(); + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject msg = response.getEntity(JSONObject.class); + JSONObject exception = msg.getJSONObject("RemoteException"); + assertEquals("incorrect number of elements", 3, exception.length()); + String message = exception.getString("message"); + String type = exception.getString("exception"); + String classname = exception.getString("javaClassName"); + WebServicesTestUtils.checkStringContains( + "exception message", + "Invalid application-state INVALID_test", + message); + WebServicesTestUtils.checkStringMatch("exception type", + "BadRequestException", type); + WebServicesTestUtils.checkStringMatch("exception classname", + "org.apache.hadoop.yarn.webapp.BadRequestException", classname); + + } finally { + rm.stop(); + } + } + @Test public void testAppsQueryStateInvalid() throws JSONException, Exception { rm.start(); @@ -280,15 +434,14 @@ public void testAppsQueryStateInvalid() throws JSONException, Exception { String message = exception.getString("message"); String type = exception.getString("exception"); String classname = exception.getString("javaClassName"); - WebServicesTestUtils - .checkStringContains( - "exception message", - "org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState.INVALID_test", - message); + WebServicesTestUtils.checkStringContains( + "exception message", + "Invalid application-state INVALID_test", + message); WebServicesTestUtils.checkStringMatch("exception type", - "IllegalArgumentException", type); + "BadRequestException", type); WebServicesTestUtils.checkStringMatch("exception classname", - "java.lang.IllegalArgumentException", classname); + "org.apache.hadoop.yarn.webapp.BadRequestException", classname); } finally { rm.stop(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm index 5f0580128d..22011124f2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm @@ -1107,10 +1107,11 @@ ResourceManager REST API's. ** Query Parameters Supported - Multiple paramters can be specified. The started and finished times have a begin and end parameter to allow you to specify ranges. For example, one could request all applications that started between 1:00am and 2:00pm on 12/19/2011 with startedTimeBegin=1324256400&startedTimeEnd=1324303200. If the Begin parameter is not specfied, it defaults to 0, and if the End parameter is not specified, it defaults to infinity. + Multiple parameters can be specified. The started and finished times have a begin and end parameter to allow you to specify ranges. For example, one could request all applications that started between 1:00am and 2:00pm on 12/19/2011 with startedTimeBegin=1324256400&startedTimeEnd=1324303200. If the Begin parameter is not specified, it defaults to 0, and if the End parameter is not specified, it defaults to infinity. ------ - * state - state of the application + * state [deprecated] - state of the application + * states - applications matching the given application states, specified as a comma-separated list. * finalStatus - the final status of the application - reported by the application itself * user - user name * queue - queue name