big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-696. Changed RMWebservice apps call to take in multiple application states. Contributed by Trevor Lorimer.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1520736 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2013-09-07 07:28:19 +00:00
parent f35983b805
commit 03ae82a311
4 changed files with 203 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -89,6 +89,9 @@ Release 2.1.1-beta - UNRELEASED
YARN-758. Augment MockNM to use multiple cores (Karthik Kambatla via
Sandy Ryza)
YARN-696. Changed RMWebservice apps call to take in multiple application
states. (Trevor Lorimer via vinodkv)
OPTIMIZATIONS
BUG FIXES

42
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
View File

@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.EnumSet;
import java.util.HashSet;
@ -231,6 +232,7 @@ public NodeInfo getNode(@PathParam("nodeId") String nodeId) {
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public AppsInfo getApps(@Context HttpServletRequest hsr,
@QueryParam("state") String stateQuery,
@QueryParam("states") Set<String> statesQuery,
@QueryParam("finalStatus") String finalStatusQuery,
@QueryParam("user") String userQuery,
@QueryParam("queue") String queueQuery,
@ -245,6 +247,7 @@ public AppsInfo getApps(@Context HttpServletRequest hsr,
boolean checkStart = false;
boolean checkEnd = false;
boolean checkAppTypes = false;
boolean checkAppStates = false;
long countNum = 0;
// set values suitable in case both of begin/end not specified
@ -321,6 +324,36 @@ public AppsInfo getApps(@Context HttpServletRequest hsr,
checkAppTypes = true;
}
String allAppStates;
RMAppState[] stateArray = RMAppState.values();
allAppStates = Arrays.toString(stateArray);
Set<String> appStates = new HashSet<String>();
// stateQuery is deprecated.
if (stateQuery != null && !stateQuery.isEmpty()) {
statesQuery.add(stateQuery);
}
if (!statesQuery.isEmpty()) {
for (String applicationState : statesQuery) {
if (applicationState != null && !applicationState.isEmpty()) {
String[] states = applicationState.split(",");
for (String state : states) {
try {
RMAppState.valueOf(state.trim());
} catch (IllegalArgumentException iae) {
throw new BadRequestException(
"Invalid application-state " + state
+ " specified. It should be one of " + allAppStates);
}
appStates.add(state.trim().toLowerCase());
}
}
}
}
if (!appStates.isEmpty()) {
checkAppStates = true;
}
final ConcurrentMap<ApplicationId, RMApp> apps = rm.getRMContext()
.getRMApps();
AppsInfo allApps = new AppsInfo();
@ -329,11 +362,10 @@ public AppsInfo getApps(@Context HttpServletRequest hsr,
if (checkCount && num == countNum) {
break;
}
if (stateQuery != null && !stateQuery.isEmpty()) {
RMAppState.valueOf(stateQuery);
if (!rmapp.getState().toString().equalsIgnoreCase(stateQuery)) {
continue;
}
if (checkAppStates
&& !appStates.contains(rmapp.getState().toString().toLowerCase())) {
continue;
}
if (finalStatusQuery != null && !finalStatusQuery.isEmpty()) {
FinalApplicationStatus.valueOf(finalStatusQuery);

167
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
View File

@ -68,6 +68,7 @@
import com.sun.jersey.api.client.ClientResponse.Status;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.core.util.MultivaluedMapImpl;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.sun.jersey.test.framework.JerseyTest;
import com.sun.jersey.test.framework.WebAppDescriptor;
@ -239,6 +240,122 @@ public void testAppsQueryState() throws JSONException, Exception {
rm.stop();
}
@Test
public void testAppsQueryStates() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
rm.submitApp(1024);
RMApp killedApp = rm.submitApp(1024);
rm.killApp(killedApp.getApplicationId());
amNodeManager.nodeHeartbeat(true);
WebResource r = resource();
MultivaluedMapImpl params = new MultivaluedMapImpl();
params.add("states", RMAppState.ACCEPTED.toString());
ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("apps").queryParams(params)
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
JSONObject json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
JSONObject apps = json.getJSONObject("apps");
assertEquals("incorrect number of elements", 1, apps.length());
JSONArray array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", 1, array.length());
assertEquals("state not equal to ACCEPTED", "ACCEPTED", array
.getJSONObject(0).getString("state"));
r = resource();
params = new MultivaluedMapImpl();
params.add("states", RMAppState.ACCEPTED.toString());
params.add("states", RMAppState.KILLED.toString());
response = r.path("ws").path("v1").path("cluster")
.path("apps").queryParams(params)
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
apps = json.getJSONObject("apps");
assertEquals("incorrect number of elements", 1, apps.length());
array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", 2, array.length());
assertTrue("both app states of ACCEPTED and KILLED are not present",
(array.getJSONObject(0).getString("state").equals("ACCEPTED") &&
array.getJSONObject(1).getString("state").equals("KILLED")) ||
(array.getJSONObject(0).getString("state").equals("KILLED") &&
array.getJSONObject(1).getString("state").equals("ACCEPTED")));
rm.stop();
}
@Test
public void testAppsQueryStatesComma() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
rm.submitApp(1024);
RMApp killedApp = rm.submitApp(1024);
rm.killApp(killedApp.getApplicationId());
amNodeManager.nodeHeartbeat(true);
WebResource r = resource();
MultivaluedMapImpl params = new MultivaluedMapImpl();
params.add("states", RMAppState.ACCEPTED.toString());
ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("apps").queryParams(params)
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
JSONObject json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
JSONObject apps = json.getJSONObject("apps");
assertEquals("incorrect number of elements", 1, apps.length());
JSONArray array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", 1, array.length());
assertEquals("state not equal to ACCEPTED", "ACCEPTED", array
.getJSONObject(0).getString("state"));
r = resource();
params = new MultivaluedMapImpl();
params.add("states", RMAppState.ACCEPTED.toString() + ","
+ RMAppState.KILLED.toString());
response = r.path("ws").path("v1").path("cluster")
.path("apps").queryParams(params)
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
apps = json.getJSONObject("apps");
assertEquals("incorrect number of elements", 1, apps.length());
array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", 2, array.length());
assertTrue("both app states of ACCEPTED and KILLED are not present",
(array.getJSONObject(0).getString("state").equals("ACCEPTED") &&
array.getJSONObject(1).getString("state").equals("KILLED")) ||
(array.getJSONObject(0).getString("state").equals("KILLED") &&
array.getJSONObject(1).getString("state").equals("ACCEPTED")));
rm.stop();
}
@Test
public void testAppsQueryStatesNone() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
rm.submitApp(1024);
amNodeManager.nodeHeartbeat(true);
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
.path("apps").queryParam("states", RMAppState.RUNNING.toString())
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
JSONObject json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
assertEquals("apps is not null", JSONObject.NULL, json.get("apps"));
rm.stop();
}
@Test
public void testAppsQueryStateNone() throws JSONException, Exception {
rm.start();
@ -257,6 +374,43 @@ public void testAppsQueryStateNone() throws JSONException, Exception {
rm.stop();
}
@Test
public void testAppsQueryStatesInvalid() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
rm.submitApp(1024);
amNodeManager.nodeHeartbeat(true);
WebResource r = resource();
try {
r.path("ws").path("v1").path("cluster").path("apps")
.queryParam("states", "INVALID_test")
.accept(MediaType.APPLICATION_JSON).get(JSONObject.class);
fail("should have thrown exception on invalid state query");
} catch (UniformInterfaceException ue) {
ClientResponse response = ue.getResponse();
assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus());
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
JSONObject msg = response.getEntity(JSONObject.class);
JSONObject exception = msg.getJSONObject("RemoteException");
assertEquals("incorrect number of elements", 3, exception.length());
String message = exception.getString("message");
String type = exception.getString("exception");
String classname = exception.getString("javaClassName");
WebServicesTestUtils.checkStringContains(
"exception message",
"Invalid application-state INVALID_test",
message);
WebServicesTestUtils.checkStringMatch("exception type",
"BadRequestException", type);
WebServicesTestUtils.checkStringMatch("exception classname",
"org.apache.hadoop.yarn.webapp.BadRequestException", classname);
} finally {
rm.stop();
}
}
@Test
public void testAppsQueryStateInvalid() throws JSONException, Exception {
rm.start();
@ -280,15 +434,14 @@ public void testAppsQueryStateInvalid() throws JSONException, Exception {
String message = exception.getString("message");
String type = exception.getString("exception");
String classname = exception.getString("javaClassName");
WebServicesTestUtils
.checkStringContains(
"exception message",
"org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState.INVALID_test",
message);
WebServicesTestUtils.checkStringContains(
"exception message",
"Invalid application-state INVALID_test",
message);
WebServicesTestUtils.checkStringMatch("exception type",
"IllegalArgumentException", type);
"BadRequestException", type);
WebServicesTestUtils.checkStringMatch("exception classname",
"java.lang.IllegalArgumentException", classname);
"org.apache.hadoop.yarn.webapp.BadRequestException", classname);
} finally {
rm.stop();

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
View File

@ -1107,10 +1107,11 @@ ResourceManager REST API's.
** Query Parameters Supported
Multiple paramters can be specified. The started and finished times have a begin and end parameter to allow you to specify ranges. For example, one could request all applications that started between 1:00am and 2:00pm on 12/19/2011 with startedTimeBegin=1324256400&startedTimeEnd=1324303200. If the Begin parameter is not specfied, it defaults to 0, and if the End parameter is not specified, it defaults to infinity.
Multiple parameters can be specified. The started and finished times have a begin and end parameter to allow you to specify ranges. For example, one could request all applications that started between 1:00am and 2:00pm on 12/19/2011 with startedTimeBegin=1324256400&startedTimeEnd=1324303200. If the Begin parameter is not specified, it defaults to 0, and if the End parameter is not specified, it defaults to infinity.
------
* state - state of the application
* state [deprecated] - state of the application
* states - applications matching the given application states, specified as a comma-separated list.
* finalStatus - the final status of the application - reported by the application itself
* user - user name
* queue - queue name