big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-2804. Fixed Timeline service to not fill the logs with JAXB bindings exceptions. Contributed by Zhijie Shen.

Browse Source
This commit is contained in:
Vinod Kumar Vavilapalli 2014-11-04 17:45:46 -08:00
parent d794f785de
commit b76179895d
5 changed files with 182 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -859,6 +859,9 @@ Release 2.6.0 - UNRELEASED
YARN-2010. Handle app-recovery failures gracefully.
(Jian He and Karthik Kambatla via kasha)
YARN-2804. Fixed Timeline service to not fill the logs with JAXB bindings
exceptions. (Zhijie Shen via vinodkv)
Release 2.5.2 - UNRELEASED
INCOMPATIBLE CHANGES

49
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEntity.java
View File

@ -31,6 +31,7 @@
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
@ -58,11 +59,11 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
private String entityId;
private Long startTime;
private List<TimelineEvent> events = new ArrayList<TimelineEvent>();
private Map<String, Set<String>> relatedEntities =
private HashMap<String, Set<String>> relatedEntities =
new HashMap<String, Set<String>>();
private Map<String, Set<Object>> primaryFilters =
private HashMap<String, Set<Object>> primaryFilters =
new HashMap<String, Set<Object>>();
private Map<String, Object> otherInfo =
private HashMap<String, Object> otherInfo =
new HashMap<String, Object>();
private String domainId;
@ -175,11 +176,17 @@ public void setEvents(List<TimelineEvent> events) {
*
* @return the related entities
*/
@XmlElement(name = "relatedentities")
public Map<String, Set<String>> getRelatedEntities() {
return relatedEntities;
}
// Required by JAXB
@Private
@XmlElement(name = "relatedentities")
public HashMap<String, Set<String>> getRelatedEntitiesJAXB() {
return relatedEntities;
}
/**
* Add an entity to the existing related entity map
*
@ -224,7 +231,11 @@ public void addRelatedEntities(Map<String, Set<String>> relatedEntities) {
*/
public void setRelatedEntities(
Map<String, Set<String>> relatedEntities) {
this.relatedEntities = relatedEntities;
if (relatedEntities != null && !(relatedEntities instanceof HashMap)) {
this.relatedEntities = new HashMap<String, Set<String>>(relatedEntities);
} else {
this.relatedEntities = (HashMap<String, Set<String>>) relatedEntities;
}
}
/**
@ -232,11 +243,17 @@ public void setRelatedEntities(
*
* @return the primary filters
*/
@XmlElement(name = "primaryfilters")
public Map<String, Set<Object>> getPrimaryFilters() {
return primaryFilters;
}
// Required by JAXB
@Private
@XmlElement(name = "primaryfilters")
public HashMap<String, Set<Object>> getPrimaryFiltersJAXB() {
return primaryFilters;
}
/**
* Add a single piece of primary filter to the existing primary filter map
*
@ -280,7 +297,11 @@ public void addPrimaryFilters(Map<String, Set<Object>> primaryFilters) {
* a map of primary filters
*/
public void setPrimaryFilters(Map<String, Set<Object>> primaryFilters) {
this.primaryFilters = primaryFilters;
if (primaryFilters != null && !(primaryFilters instanceof HashMap)) {
this.primaryFilters = new HashMap<String, Set<Object>>(primaryFilters);
} else {
this.primaryFilters = (HashMap<String, Set<Object>>) primaryFilters;
}
}
/**
@ -288,11 +309,17 @@ public void setPrimaryFilters(Map<String, Set<Object>> primaryFilters) {
*
* @return the other information of the entity
*/
@XmlElement(name = "otherinfo")
public Map<String, Object> getOtherInfo() {
return otherInfo;
}
// Required by JAXB
@Private
@XmlElement(name = "otherinfo")
public HashMap<String, Object> getOtherInfoJAXB() {
return otherInfo;
}
/**
* Add one piece of other information of the entity to the existing other info
* map
@ -323,7 +350,11 @@ public void addOtherInfo(Map<String, Object> otherInfo) {
* a map of other information
*/
public void setOtherInfo(Map<String, Object> otherInfo) {
this.otherInfo = otherInfo;
if (otherInfo != null && !(otherInfo instanceof HashMap)) {
this.otherInfo = new HashMap<String, Object>(otherInfo);
} else {
this.otherInfo = (HashMap<String, Object>) otherInfo;
}
}
/**

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEvent.java
View File

@ -26,6 +26,7 @@
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
@ -43,7 +44,7 @@ public class TimelineEvent implements Comparable<TimelineEvent> {
private long timestamp;
private String eventType;
private Map<String, Object> eventInfo = new HashMap<String, Object>();
private HashMap<String, Object> eventInfo = new HashMap<String, Object>();
public TimelineEvent() {
}
@ -93,11 +94,17 @@ public void setEventType(String eventType) {
*
* @return the information of the event
*/
@XmlElement(name = "eventinfo")
public Map<String, Object> getEventInfo() {
return eventInfo;
}
// Required by JAXB
@Private
@XmlElement(name = "eventinfo")
public HashMap<String, Object> getEventInfoJAXB() {
return eventInfo;
}
/**
* Add one piece of the information of the event to the existing information
* map
@ -128,7 +135,11 @@ public void addEventInfo(Map<String, Object> eventInfo) {
* a map of of the information of the event
*/
public void setEventInfo(Map<String, Object> eventInfo) {
this.eventInfo = eventInfo;
if (eventInfo != null && !(eventInfo instanceof HashMap)) {
this.eventInfo = new HashMap<String, Object>(eventInfo);
} else {
this.eventInfo = (HashMap<String, Object>) eventInfo;
}
}
@Override

128
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java
View File

@ -19,19 +19,19 @@
package org.apache.hadoop.yarn.api.records.timeline;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import org.junit.Assert;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.WeakHashMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.junit.Assert;
import org.junit.Test;
public class TestTimelineRecords {
@ -202,4 +202,118 @@ public void testTimelineDomain() throws Exception {
}
}
@Test
public void testMapInterfaceOrTimelineRecords() throws Exception {
TimelineEntity entity = new TimelineEntity();
List<Map<String, Set<Object>>> primaryFiltersList =
new ArrayList<Map<String, Set<Object>>>();
primaryFiltersList.add(
Collections.singletonMap("pkey", Collections.singleton((Object) "pval")));
Map<String, Set<Object>> primaryFilters = new TreeMap<String, Set<Object>>();
primaryFilters.put("pkey1", Collections.singleton((Object) "pval1"));
primaryFilters.put("pkey2", Collections.singleton((Object) "pval2"));
primaryFiltersList.add(primaryFilters);
entity.setPrimaryFilters(null);
for (Map<String, Set<Object>> primaryFiltersToSet : primaryFiltersList) {
entity.setPrimaryFilters(primaryFiltersToSet);
assertPrimaryFilters(entity);
Map<String, Set<Object>> primaryFiltersToAdd =
new WeakHashMap<String, Set<Object>>();
primaryFiltersToAdd.put("pkey3", Collections.singleton((Object) "pval3"));
entity.addPrimaryFilters(primaryFiltersToAdd);
assertPrimaryFilters(entity);
}
List<Map<String, Set<String>>> relatedEntitiesList =
new ArrayList<Map<String, Set<String>>>();
relatedEntitiesList.add(
Collections.singletonMap("rkey", Collections.singleton("rval")));
Map<String, Set<String>> relatedEntities = new TreeMap<String, Set<String>>();
relatedEntities.put("rkey1", Collections.singleton("rval1"));
relatedEntities.put("rkey2", Collections.singleton("rval2"));
relatedEntitiesList.add(relatedEntities);
entity.setRelatedEntities(null);
for (Map<String, Set<String>> relatedEntitiesToSet : relatedEntitiesList) {
entity.setRelatedEntities(relatedEntitiesToSet);
assertRelatedEntities(entity);
Map<String, Set<String>> relatedEntitiesToAdd =
new WeakHashMap<String, Set<String>>();
relatedEntitiesToAdd.put("rkey3", Collections.singleton("rval3"));
entity.addRelatedEntities(relatedEntitiesToAdd);
assertRelatedEntities(entity);
}
List<Map<String, Object>> otherInfoList =
new ArrayList<Map<String, Object>>();
otherInfoList.add(Collections.singletonMap("okey", (Object) "oval"));
Map<String, Object> otherInfo = new TreeMap<String, Object>();
otherInfo.put("okey1", "oval1");
otherInfo.put("okey2", "oval2");
otherInfoList.add(otherInfo);
entity.setOtherInfo(null);
for (Map<String, Object> otherInfoToSet : otherInfoList) {
entity.setOtherInfo(otherInfoToSet);
assertOtherInfo(entity);
Map<String, Object> otherInfoToAdd = new WeakHashMap<String, Object>();
otherInfoToAdd.put("okey3", "oval3");
entity.addOtherInfo(otherInfoToAdd);
assertOtherInfo(entity);
}
TimelineEvent event = new TimelineEvent();
List<Map<String, Object>> eventInfoList =
new ArrayList<Map<String, Object>>();
eventInfoList.add(Collections.singletonMap("ekey", (Object) "eval"));
Map<String, Object> eventInfo = new TreeMap<String, Object>();
eventInfo.put("ekey1", "eval1");
eventInfo.put("ekey2", "eval2");
eventInfoList.add(eventInfo);
event.setEventInfo(null);
for (Map<String, Object> eventInfoToSet : eventInfoList) {
event.setEventInfo(eventInfoToSet);
assertEventInfo(event);
Map<String, Object> eventInfoToAdd = new WeakHashMap<String, Object>();
eventInfoToAdd.put("ekey3", "eval3");
event.addEventInfo(eventInfoToAdd);
assertEventInfo(event);
}
}
private static void assertPrimaryFilters(TimelineEntity entity) {
Assert.assertNotNull(entity.getPrimaryFilters());
Assert.assertNotNull(entity.getPrimaryFiltersJAXB());
Assert.assertTrue(entity.getPrimaryFilters() instanceof HashMap);
Assert.assertTrue(entity.getPrimaryFiltersJAXB() instanceof HashMap);
Assert.assertEquals(
entity.getPrimaryFilters(), entity.getPrimaryFiltersJAXB());
}
private static void assertRelatedEntities(TimelineEntity entity) {
Assert.assertNotNull(entity.getRelatedEntities());
Assert.assertNotNull(entity.getRelatedEntitiesJAXB());
Assert.assertTrue(entity.getRelatedEntities() instanceof HashMap);
Assert.assertTrue(entity.getRelatedEntitiesJAXB() instanceof HashMap);
Assert.assertEquals(
entity.getRelatedEntities(), entity.getRelatedEntitiesJAXB());
}
private static void assertOtherInfo(TimelineEntity entity) {
Assert.assertNotNull(entity.getOtherInfo());
Assert.assertNotNull(entity.getOtherInfoJAXB());
Assert.assertTrue(entity.getOtherInfo() instanceof HashMap);
Assert.assertTrue(entity.getOtherInfoJAXB() instanceof HashMap);
Assert.assertEquals(entity.getOtherInfo(), entity.getOtherInfoJAXB());
}
private static void assertEventInfo(TimelineEvent event) {
Assert.assertNotNull(event);
Assert.assertNotNull(event.getEventInfoJAXB());
Assert.assertTrue(event.getEventInfo() instanceof HashMap);
Assert.assertTrue(event.getEventInfoJAXB() instanceof HashMap);
Assert.assertEquals(event.getEventInfo(), event.getEventInfoJAXB());
}
}

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
View File

@ -42,7 +42,6 @@
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
@ -53,11 +52,11 @@
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents;
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.timeline.EntityIdentifier;
@ -272,7 +271,7 @@ public TimelinePutResponse postEntities(
@PUT
@Path("/domain")
@Consumes({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
public Response putDomain(
public TimelinePutResponse putDomain(
@Context HttpServletRequest req,
@Context HttpServletResponse res,
TimelineDomain domain) {
@ -295,7 +294,7 @@ public Response putDomain(
throw new WebApplicationException(e,
Response.Status.INTERNAL_SERVER_ERROR);
}
return Response.status(Status.OK).build();
return new TimelinePutResponse();
}
/**