MAPREDUCE-3553. Add support for data returned when exceptions thrown from web service apis to be in either xml or in JSON. (Thomas Graves via mahadev)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1230330 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Mahadev Konar 2012-01-11 23:59:23 +00:00
parent 0510e0659c
commit 5f79f180f6
11 changed files with 372 additions and 59 deletions

View File

@ -178,6 +178,9 @@ Release 0.23.1 - Unreleased
Improved the earlier patch to not to JobHistoryServer repeatedly.
(Anupam Seth via vinodkv)
MAPREDUCE-3553. Add support for data returned when exceptions thrown from web
service apis to be in either xml or in JSON. (Thomas Graves via mahadev)
OPTIMIZATIONS
MAPREDUCE-3567. Extraneous JobConf objects in AM heap. (Vinod Kumar

View File

@ -49,6 +49,7 @@
import org.apache.hadoop.mapreduce.v2.app.webapp.dao.TaskCounterInfo;
import org.apache.hadoop.mapreduce.v2.app.webapp.dao.TaskInfo;
import org.apache.hadoop.mapreduce.v2.app.webapp.dao.TasksInfo;
import org.apache.hadoop.yarn.webapp.RemoteExceptionData;
@Singleton
@Provider
@ -64,7 +65,7 @@ public class JAXBContextResolver implements ContextResolver<JAXBContext> {
JobCounterInfo.class, TaskCounterInfo.class, CounterGroupInfo.class,
JobInfo.class, JobsInfo.class, ReduceTaskAttemptInfo.class,
TaskAttemptInfo.class, TaskInfo.class, TasksInfo.class,
TaskAttemptsInfo.class, ConfEntryInfo.class};
TaskAttemptsInfo.class, ConfEntryInfo.class, RemoteExceptionData.class};
public JAXBContextResolver() throws Exception {
this.types = new HashSet<Class>(Arrays.asList(cTypes));

View File

@ -345,6 +345,29 @@ public void testJobIdNonExist() throws JSONException, Exception {
public void testJobIdInvalid() throws JSONException, Exception {
WebResource r = resource();
try {
r.path("ws").path("v1").path("mapreduce").path("jobs").path("job_foo")
.accept(MediaType.APPLICATION_JSON).get(JSONObject.class);
fail("should have thrown exception on invalid uri");
} catch (UniformInterfaceException ue) {
ClientResponse response = ue.getResponse();
assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus());
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
JSONObject msg = response.getEntity(JSONObject.class);
JSONObject exception = msg.getJSONObject("RemoteException");
assertEquals("incorrect number of elements", 3, exception.length());
String message = exception.getString("message");
String type = exception.getString("exception");
String classname = exception.getString("javaClassName");
verifyJobIdInvalid(message, type, classname);
}
}
// verify the exception output default is JSON
@Test
public void testJobIdInvalidDefault() throws JSONException, Exception {
WebResource r = resource();
try {
r.path("ws").path("v1").path("mapreduce").path("jobs").path("job_foo")
.get(JSONObject.class);
@ -359,15 +382,49 @@ public void testJobIdInvalid() throws JSONException, Exception {
String message = exception.getString("message");
String type = exception.getString("exception");
String classname = exception.getString("javaClassName");
WebServicesTestUtils.checkStringMatch("exception message",
"For input string: \"foo\"", message);
WebServicesTestUtils.checkStringMatch("exception type",
"NumberFormatException", type);
WebServicesTestUtils.checkStringMatch("exception classname",
"java.lang.NumberFormatException", classname);
verifyJobIdInvalid(message, type, classname);
}
}
// test that the exception output works in XML
@Test
public void testJobIdInvalidXML() throws JSONException, Exception {
WebResource r = resource();
try {
r.path("ws").path("v1").path("mapreduce").path("jobs").path("job_foo")
.accept(MediaType.APPLICATION_XML).get(JSONObject.class);
fail("should have thrown exception on invalid uri");
} catch (UniformInterfaceException ue) {
ClientResponse response = ue.getResponse();
assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus());
assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
String msg = response.getEntity(String.class);
System.out.println(msg);
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
DocumentBuilder db = dbf.newDocumentBuilder();
InputSource is = new InputSource();
is.setCharacterStream(new StringReader(msg));
Document dom = db.parse(is);
NodeList nodes = dom.getElementsByTagName("RemoteException");
Element element = (Element) nodes.item(0);
String message = WebServicesTestUtils.getXmlString(element, "message");
String type = WebServicesTestUtils.getXmlString(element, "exception");
String classname = WebServicesTestUtils.getXmlString(element,
"javaClassName");
verifyJobIdInvalid(message, type, classname);
}
}
private void verifyJobIdInvalid(String message, String type, String classname) {
WebServicesTestUtils.checkStringMatch("exception message",
"For input string: \"foo\"", message);
WebServicesTestUtils.checkStringMatch("exception type",
"NumberFormatException", type);
WebServicesTestUtils.checkStringMatch("exception classname",
"java.lang.NumberFormatException", classname);
}
@Test
public void testJobIdInvalidBogus() throws JSONException, Exception {
WebResource r = resource();

View File

@ -49,6 +49,7 @@
import org.apache.hadoop.mapreduce.v2.hs.webapp.dao.HistoryInfo;
import org.apache.hadoop.mapreduce.v2.hs.webapp.dao.JobInfo;
import org.apache.hadoop.mapreduce.v2.hs.webapp.dao.JobsInfo;
import org.apache.hadoop.yarn.webapp.RemoteExceptionData;
@Singleton
@Provider
@ -64,7 +65,8 @@ public class JAXBContextResolver implements ContextResolver<JAXBContext> {
JobTaskAttemptCounterInfo.class, TaskCounterInfo.class,
JobCounterInfo.class, ReduceTaskAttemptInfo.class, TaskAttemptInfo.class,
TaskAttemptsInfo.class, CounterGroupInfo.class,
TaskCounterGroupInfo.class, AMAttemptInfo.class, AMAttemptsInfo.class };
TaskCounterGroupInfo.class, AMAttemptInfo.class, AMAttemptsInfo.class,
RemoteExceptionData.class };
public JAXBContextResolver() throws Exception {
this.types = new HashSet<Class>(Arrays.asList(cTypes));

View File

@ -392,6 +392,31 @@ public void testJobIdNonExist() throws JSONException, Exception {
public void testJobIdInvalid() throws JSONException, Exception {
WebResource r = resource();
try {
r.path("ws").path("v1").path("history").path("mapreduce").path("jobs")
.path("job_foo").accept(MediaType.APPLICATION_JSON)
.get(JSONObject.class);
fail("should have thrown exception on invalid uri");
} catch (UniformInterfaceException ue) {
ClientResponse response = ue.getResponse();
assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus());
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
JSONObject msg = response.getEntity(JSONObject.class);
JSONObject exception = msg.getJSONObject("RemoteException");
assertEquals("incorrect number of elements", 3, exception.length());
String message = exception.getString("message");
String type = exception.getString("exception");
String classname = exception.getString("javaClassName");
verifyJobIdInvalid(message, type, classname);
}
}
// verify the exception output default is JSON
@Test
public void testJobIdInvalidDefault() throws JSONException, Exception {
WebResource r = resource();
try {
r.path("ws").path("v1").path("history").path("mapreduce").path("jobs")
.path("job_foo").get(JSONObject.class);
@ -406,15 +431,50 @@ public void testJobIdInvalid() throws JSONException, Exception {
String message = exception.getString("message");
String type = exception.getString("exception");
String classname = exception.getString("javaClassName");
WebServicesTestUtils.checkStringMatch("exception message",
"For input string: \"foo\"", message);
WebServicesTestUtils.checkStringMatch("exception type",
"NumberFormatException", type);
WebServicesTestUtils.checkStringMatch("exception classname",
"java.lang.NumberFormatException", classname);
verifyJobIdInvalid(message, type, classname);
}
}
// test that the exception output works in XML
@Test
public void testJobIdInvalidXML() throws JSONException, Exception {
WebResource r = resource();
try {
r.path("ws").path("v1").path("history").path("mapreduce").path("jobs")
.path("job_foo").accept(MediaType.APPLICATION_XML)
.get(JSONObject.class);
fail("should have thrown exception on invalid uri");
} catch (UniformInterfaceException ue) {
ClientResponse response = ue.getResponse();
assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus());
assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
String msg = response.getEntity(String.class);
System.out.println(msg);
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
DocumentBuilder db = dbf.newDocumentBuilder();
InputSource is = new InputSource();
is.setCharacterStream(new StringReader(msg));
Document dom = db.parse(is);
NodeList nodes = dom.getElementsByTagName("RemoteException");
Element element = (Element) nodes.item(0);
String message = WebServicesTestUtils.getXmlString(element, "message");
String type = WebServicesTestUtils.getXmlString(element, "exception");
String classname = WebServicesTestUtils.getXmlString(element,
"javaClassName");
verifyJobIdInvalid(message, type, classname);
}
}
private void verifyJobIdInvalid(String message, String type, String classname) {
WebServicesTestUtils.checkStringMatch("exception message",
"For input string: \"foo\"", message);
WebServicesTestUtils.checkStringMatch("exception type",
"NumberFormatException", type);
WebServicesTestUtils.checkStringMatch("exception classname",
"java.lang.NumberFormatException", classname);
}
@Test
public void testJobIdInvalidBogus() throws JSONException, Exception {
WebResource r = resource();

View File

@ -19,12 +19,9 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Map;
import java.util.TreeMap;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.ext.ExceptionMapper;
import javax.ws.rs.ext.Provider;
@ -33,19 +30,12 @@
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.security.authorize.AuthorizationException;
import org.mortbay.util.ajax.JSON;
import com.google.inject.Singleton;
/**
* Handle webservices jersey exceptions and create json response in the format:
* { "RemoteException" :
* {
* "exception" : <exception type>,
* "javaClassName" : <classname of exception>,
* "message" : <error message from exception>
* }
* }
* Handle webservices jersey exceptions and create json or xml response
* with the ExceptionData.
*/
@Singleton
@Provider
@ -100,16 +90,11 @@ public Response toResponse(Exception e) {
s = Response.Status.INTERNAL_SERVER_ERROR;
}
// convert to json
final Map<String, Object> m = new TreeMap<String, Object>();
m.put("exception", e.getClass().getSimpleName());
m.put("message", e.getMessage());
m.put("javaClassName", e.getClass().getName());
final Map<String, Object> m2 = new TreeMap<String, Object>();
m2.put(RemoteException.class.getSimpleName(), m);
final String js = JSON.toString(m2);
// let jaxb handle marshalling data out in the same format requested
RemoteExceptionData exception = new RemoteExceptionData(e.getClass().getSimpleName(),
e.getMessage(), e.getClass().getName());
return Response.status(s).type(MediaType.APPLICATION_JSON).entity(js)
return Response.status(s).entity(exception)
.build();
}
}

View File

@ -0,0 +1,63 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.webapp;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
/**
* Contains the exception information from an exception thrown
* by the web service REST API's.
* Fields include:
* exception - exception type
* javaClassName - java class name of the exception
* message - a detailed message explaining the exception
*
*/
@XmlRootElement(name = "RemoteException")
@XmlAccessorType(XmlAccessType.FIELD)
public class RemoteExceptionData {
private String exception;
private String message;
private String javaClassName;
public RemoteExceptionData() {
}
public RemoteExceptionData(String excep, String message, String className) {
this.exception = excep;
this.message = message;
this.javaClassName = className;
}
public String getException() {
return exception;
}
public String getMessage() {
return message;
}
public String getJavaClassName() {
return javaClassName;
}
}

View File

@ -35,6 +35,7 @@
import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.ContainerInfo;
import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.ContainersInfo;
import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NodeInfo;
import org.apache.hadoop.yarn.webapp.RemoteExceptionData;
@Singleton
@Provider
@ -42,19 +43,20 @@ public class JAXBContextResolver implements ContextResolver<JAXBContext> {
private JAXBContext context;
private final Set<Class> types;
// you have to specify all the dao classes here
private final Class[] cTypes = {AppInfo.class, AppsInfo.class,
ContainerInfo.class, ContainersInfo.class, NodeInfo.class};
private final Class[] cTypes = {AppInfo.class, AppsInfo.class,
ContainerInfo.class, ContainersInfo.class, NodeInfo.class,
RemoteExceptionData.class};
public JAXBContextResolver() throws Exception {
this.types = new HashSet<Class>(Arrays.asList(cTypes));
// sets the json configuration so that the json output looks like
// sets the json configuration so that the json output looks like
// the xml output
this.context = new JSONJAXBContext(JSONConfiguration.natural().
rootUnwrapping(false).build(), cTypes);
}
@Override
public JAXBContext getContext(Class<?> objectType) {
return (types.contains(objectType)) ? context : null;

View File

@ -382,18 +382,91 @@ public void testNodeAppsStateInvalid() throws JSONException, Exception {
String message = exception.getString("message");
String type = exception.getString("exception");
String classname = exception.getString("javaClassName");
WebServicesTestUtils
.checkStringMatch(
"exception message",
"No enum const class org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState.FOO_STATE",
message);
WebServicesTestUtils.checkStringMatch("exception type",
"IllegalArgumentException", type);
WebServicesTestUtils.checkStringMatch("exception classname",
"java.lang.IllegalArgumentException", classname);
verifyStatInvalidException(message, type, classname);
}
}
// verify the exception object default format is JSON
@Test
public void testNodeAppsStateInvalidDefault() throws JSONException, Exception {
WebResource r = resource();
Application app = new MockApp(1);
nmContext.getApplications().put(app.getAppId(), app);
addAppContainers(app);
Application app2 = new MockApp("foo", 1234, 2);
nmContext.getApplications().put(app2.getAppId(), app2);
addAppContainers(app2);
try {
r.path("ws").path("v1").path("node").path("apps")
.queryParam("state", "FOO_STATE").get(JSONObject.class);
fail("should have thrown exception on invalid user query");
} catch (UniformInterfaceException ue) {
ClientResponse response = ue.getResponse();
assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus());
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
JSONObject msg = response.getEntity(JSONObject.class);
JSONObject exception = msg.getJSONObject("RemoteException");
assertEquals("incorrect number of elements", 3, exception.length());
String message = exception.getString("message");
String type = exception.getString("exception");
String classname = exception.getString("javaClassName");
verifyStatInvalidException(message, type, classname);
}
}
// test that the exception output also returns XML
@Test
public void testNodeAppsStateInvalidXML() throws JSONException, Exception {
WebResource r = resource();
Application app = new MockApp(1);
nmContext.getApplications().put(app.getAppId(), app);
addAppContainers(app);
Application app2 = new MockApp("foo", 1234, 2);
nmContext.getApplications().put(app2.getAppId(), app2);
addAppContainers(app2);
try {
r.path("ws").path("v1").path("node").path("apps")
.queryParam("state", "FOO_STATE").accept(MediaType.APPLICATION_XML)
.get(JSONObject.class);
fail("should have thrown exception on invalid user query");
} catch (UniformInterfaceException ue) {
ClientResponse response = ue.getResponse();
assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus());
assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
String msg = response.getEntity(String.class);
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
DocumentBuilder db = dbf.newDocumentBuilder();
InputSource is = new InputSource();
is.setCharacterStream(new StringReader(msg));
Document dom = db.parse(is);
NodeList nodes = dom.getElementsByTagName("RemoteException");
Element element = (Element) nodes.item(0);
String message = WebServicesTestUtils.getXmlString(element, "message");
String type = WebServicesTestUtils.getXmlString(element, "exception");
String classname = WebServicesTestUtils.getXmlString(element,
"javaClassName");
verifyStatInvalidException(message, type, classname);
}
}
private void verifyStatInvalidException(String message, String type,
String classname) {
WebServicesTestUtils
.checkStringMatch(
"exception message",
"No enum const class org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState.FOO_STATE",
message);
WebServicesTestUtils.checkStringMatch("exception type",
"IllegalArgumentException", type);
WebServicesTestUtils.checkStringMatch("exception classname",
"java.lang.IllegalArgumentException", classname);
}
@Test
public void testNodeSingleApps() throws JSONException, Exception {
testNodeSingleAppHelper(MediaType.APPLICATION_JSON);

View File

@ -42,6 +42,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.UserMetricsInfo;
import org.apache.hadoop.yarn.webapp.RemoteExceptionData;
@Singleton
@Provider
@ -55,7 +56,8 @@ public class JAXBContextResolver implements ContextResolver<JAXBContext> {
CapacitySchedulerQueueInfo.class, FifoSchedulerInfo.class,
SchedulerTypeInfo.class, NodeInfo.class, UserMetricsInfo.class,
CapacitySchedulerInfo.class, ClusterMetricsInfo.class,
SchedulerInfo.class, AppsInfo.class, NodesInfo.class };
SchedulerInfo.class, AppsInfo.class, NodesInfo.class,
RemoteExceptionData.class};
public JAXBContextResolver() throws Exception {
this.types = new HashSet<Class>(Arrays.asList(cTypes));

View File

@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.StringReader;
@ -404,20 +405,84 @@ public void testNonexistNode() throws JSONException, Exception {
String message = exception.getString("message");
String type = exception.getString("exception");
String classname = exception.getString("javaClassName");
WebServicesTestUtils
.checkStringMatch("exception message",
"java.lang.Exception: nodeId, node_invalid:99, is not found",
message);
WebServicesTestUtils.checkStringMatch("exception type",
"NotFoundException", type);
WebServicesTestUtils.checkStringMatch("exception classname",
"org.apache.hadoop.yarn.webapp.NotFoundException", classname);
verifyNonexistNodeException(message, type, classname);
} finally {
rm.stop();
}
}
// test that the exception output defaults to JSON
@Test
public void testNonexistNodeDefault() throws JSONException, Exception {
rm.registerNode("h1:1234", 5120);
rm.registerNode("h2:1235", 5121);
WebResource r = resource();
try {
r.path("ws").path("v1").path("cluster").path("nodes")
.path("node_invalid:99").get(JSONObject.class);
fail("should have thrown exception on non-existent nodeid");
} catch (UniformInterfaceException ue) {
ClientResponse response = ue.getResponse();
assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
JSONObject msg = response.getEntity(JSONObject.class);
JSONObject exception = msg.getJSONObject("RemoteException");
assertEquals("incorrect number of elements", 3, exception.length());
String message = exception.getString("message");
String type = exception.getString("exception");
String classname = exception.getString("javaClassName");
verifyNonexistNodeException(message, type, classname);
} finally {
rm.stop();
}
}
// test that the exception output works in XML
@Test
public void testNonexistNodeXML() throws JSONException, Exception {
rm.registerNode("h1:1234", 5120);
rm.registerNode("h2:1235", 5121);
WebResource r = resource();
try {
r.path("ws").path("v1").path("cluster").path("nodes")
.path("node_invalid:99").accept(MediaType.APPLICATION_XML)
.get(JSONObject.class);
fail("should have thrown exception on non-existent nodeid");
} catch (UniformInterfaceException ue) {
ClientResponse response = ue.getResponse();
assertEquals(Status.NOT_FOUND, response.getClientResponseStatus());
assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
String msg = response.getEntity(String.class);
System.out.println(msg);
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
DocumentBuilder db = dbf.newDocumentBuilder();
InputSource is = new InputSource();
is.setCharacterStream(new StringReader(msg));
Document dom = db.parse(is);
NodeList nodes = dom.getElementsByTagName("RemoteException");
Element element = (Element) nodes.item(0);
String message = WebServicesTestUtils.getXmlString(element, "message");
String type = WebServicesTestUtils.getXmlString(element, "exception");
String classname = WebServicesTestUtils.getXmlString(element,
"javaClassName");
verifyNonexistNodeException(message, type, classname);
} finally {
rm.stop();
}
}
private void verifyNonexistNodeException(String message, String type, String classname) {
assertTrue("exception message incorrect",
"java.lang.Exception: nodeId, node_invalid:99, is not found"
.matches(message));
assertTrue("exception type incorrect", "NotFoundException".matches(type));
assertTrue("exception className incorrect",
"org.apache.hadoop.yarn.webapp.NotFoundException".matches(classname));
}
@Test
public void testInvalidNode() throws JSONException, Exception {
rm.registerNode("h1:1234", 5120);