From b76179895dd2ef4d56e8de31e9f673375faa2afa Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Tue, 4 Nov 2014 17:45:46 -0800 Subject: [PATCH] YARN-2804. Fixed Timeline service to not fill the logs with JAXB bindings exceptions. Contributed by Zhijie Shen. --- hadoop-yarn-project/CHANGES.txt | 3 + .../api/records/timeline/TimelineEntity.java | 49 +++++-- .../api/records/timeline/TimelineEvent.java | 17 ++- .../records/timeline/TestTimelineRecords.java | 128 +++++++++++++++++- .../timeline/webapp/TimelineWebServices.java | 9 +- 5 files changed, 182 insertions(+), 24 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index f9fcf5b3eb..b5aec42187 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -859,6 +859,9 @@ Release 2.6.0 - UNRELEASED YARN-2010. Handle app-recovery failures gracefully. (Jian He and Karthik Kambatla via kasha) + YARN-2804. Fixed Timeline service to not fill the logs with JAXB bindings + exceptions. (Zhijie Shen via vinodkv) + Release 2.5.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEntity.java index d66c253f85..03ed4af5d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEntity.java @@ -31,6 +31,7 @@ import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; @@ -58,11 +59,11 @@ public class TimelineEntity implements Comparable { private String entityId; private Long startTime; private List events = new ArrayList(); - private Map> relatedEntities = + private HashMap> relatedEntities = new HashMap>(); - private Map> primaryFilters = + private HashMap> primaryFilters = new HashMap>(); - private Map otherInfo = + private HashMap otherInfo = new HashMap(); private String domainId; @@ -175,11 +176,17 @@ public void setEvents(List events) { * * @return the related entities */ - @XmlElement(name = "relatedentities") public Map> getRelatedEntities() { return relatedEntities; } + // Required by JAXB + @Private + @XmlElement(name = "relatedentities") + public HashMap> getRelatedEntitiesJAXB() { + return relatedEntities; + } + /** * Add an entity to the existing related entity map * @@ -224,7 +231,11 @@ public void addRelatedEntities(Map> relatedEntities) { */ public void setRelatedEntities( Map> relatedEntities) { - this.relatedEntities = relatedEntities; + if (relatedEntities != null && !(relatedEntities instanceof HashMap)) { + this.relatedEntities = new HashMap>(relatedEntities); + } else { + this.relatedEntities = (HashMap>) relatedEntities; + } } /** @@ -232,11 +243,17 @@ public void setRelatedEntities( * * @return the primary filters */ - @XmlElement(name = "primaryfilters") public Map> getPrimaryFilters() { return primaryFilters; } + // Required by JAXB + @Private + @XmlElement(name = "primaryfilters") + public HashMap> getPrimaryFiltersJAXB() { + return primaryFilters; + } + /** * Add a single piece of primary filter to the existing primary filter map * @@ -280,7 +297,11 @@ public void addPrimaryFilters(Map> primaryFilters) { * a map of primary filters */ public void setPrimaryFilters(Map> primaryFilters) { - this.primaryFilters = primaryFilters; + if (primaryFilters != null && !(primaryFilters instanceof HashMap)) { + this.primaryFilters = new HashMap>(primaryFilters); + } else { + this.primaryFilters = (HashMap>) primaryFilters; + } } /** @@ -288,11 +309,17 @@ public void setPrimaryFilters(Map> primaryFilters) { * * @return the other information of the entity */ - @XmlElement(name = "otherinfo") public Map getOtherInfo() { return otherInfo; } + // Required by JAXB + @Private + @XmlElement(name = "otherinfo") + public HashMap getOtherInfoJAXB() { + return otherInfo; + } + /** * Add one piece of other information of the entity to the existing other info * map @@ -323,7 +350,11 @@ public void addOtherInfo(Map otherInfo) { * a map of other information */ public void setOtherInfo(Map otherInfo) { - this.otherInfo = otherInfo; + if (otherInfo != null && !(otherInfo instanceof HashMap)) { + this.otherInfo = new HashMap(otherInfo); + } else { + this.otherInfo = (HashMap) otherInfo; + } } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEvent.java index aa4953820a..0a09e785ef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEvent.java @@ -26,6 +26,7 @@ import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; @@ -43,7 +44,7 @@ public class TimelineEvent implements Comparable { private long timestamp; private String eventType; - private Map eventInfo = new HashMap(); + private HashMap eventInfo = new HashMap(); public TimelineEvent() { } @@ -93,11 +94,17 @@ public void setEventType(String eventType) { * * @return the information of the event */ - @XmlElement(name = "eventinfo") public Map getEventInfo() { return eventInfo; } + // Required by JAXB + @Private + @XmlElement(name = "eventinfo") + public HashMap getEventInfoJAXB() { + return eventInfo; + } + /** * Add one piece of the information of the event to the existing information * map @@ -128,7 +135,11 @@ public void addEventInfo(Map eventInfo) { * a map of of the information of the event */ public void setEventInfo(Map eventInfo) { - this.eventInfo = eventInfo; + if (eventInfo != null && !(eventInfo instanceof HashMap)) { + this.eventInfo = new HashMap(eventInfo); + } else { + this.eventInfo = (HashMap) eventInfo; + } } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java index 2b59ff564c..9d16edb716 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java @@ -19,19 +19,19 @@ package org.apache.hadoop.yarn.api.records.timeline; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.List; - -import org.junit.Assert; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.WeakHashMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; -import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; -import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; -import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents; -import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; +import org.junit.Assert; import org.junit.Test; public class TestTimelineRecords { @@ -202,4 +202,118 @@ public void testTimelineDomain() throws Exception { } } + @Test + public void testMapInterfaceOrTimelineRecords() throws Exception { + TimelineEntity entity = new TimelineEntity(); + List>> primaryFiltersList = + new ArrayList>>(); + primaryFiltersList.add( + Collections.singletonMap("pkey", Collections.singleton((Object) "pval"))); + Map> primaryFilters = new TreeMap>(); + primaryFilters.put("pkey1", Collections.singleton((Object) "pval1")); + primaryFilters.put("pkey2", Collections.singleton((Object) "pval2")); + primaryFiltersList.add(primaryFilters); + entity.setPrimaryFilters(null); + for (Map> primaryFiltersToSet : primaryFiltersList) { + entity.setPrimaryFilters(primaryFiltersToSet); + assertPrimaryFilters(entity); + + Map> primaryFiltersToAdd = + new WeakHashMap>(); + primaryFiltersToAdd.put("pkey3", Collections.singleton((Object) "pval3")); + entity.addPrimaryFilters(primaryFiltersToAdd); + assertPrimaryFilters(entity); + } + + List>> relatedEntitiesList = + new ArrayList>>(); + relatedEntitiesList.add( + Collections.singletonMap("rkey", Collections.singleton("rval"))); + Map> relatedEntities = new TreeMap>(); + relatedEntities.put("rkey1", Collections.singleton("rval1")); + relatedEntities.put("rkey2", Collections.singleton("rval2")); + relatedEntitiesList.add(relatedEntities); + entity.setRelatedEntities(null); + for (Map> relatedEntitiesToSet : relatedEntitiesList) { + entity.setRelatedEntities(relatedEntitiesToSet); + assertRelatedEntities(entity); + + Map> relatedEntitiesToAdd = + new WeakHashMap>(); + relatedEntitiesToAdd.put("rkey3", Collections.singleton("rval3")); + entity.addRelatedEntities(relatedEntitiesToAdd); + assertRelatedEntities(entity); + } + + List> otherInfoList = + new ArrayList>(); + otherInfoList.add(Collections.singletonMap("okey", (Object) "oval")); + Map otherInfo = new TreeMap(); + otherInfo.put("okey1", "oval1"); + otherInfo.put("okey2", "oval2"); + otherInfoList.add(otherInfo); + entity.setOtherInfo(null); + for (Map otherInfoToSet : otherInfoList) { + entity.setOtherInfo(otherInfoToSet); + assertOtherInfo(entity); + + Map otherInfoToAdd = new WeakHashMap(); + otherInfoToAdd.put("okey3", "oval3"); + entity.addOtherInfo(otherInfoToAdd); + assertOtherInfo(entity); + } + + TimelineEvent event = new TimelineEvent(); + List> eventInfoList = + new ArrayList>(); + eventInfoList.add(Collections.singletonMap("ekey", (Object) "eval")); + Map eventInfo = new TreeMap(); + eventInfo.put("ekey1", "eval1"); + eventInfo.put("ekey2", "eval2"); + eventInfoList.add(eventInfo); + event.setEventInfo(null); + for (Map eventInfoToSet : eventInfoList) { + event.setEventInfo(eventInfoToSet); + assertEventInfo(event); + + Map eventInfoToAdd = new WeakHashMap(); + eventInfoToAdd.put("ekey3", "eval3"); + event.addEventInfo(eventInfoToAdd); + assertEventInfo(event); + } + } + + private static void assertPrimaryFilters(TimelineEntity entity) { + Assert.assertNotNull(entity.getPrimaryFilters()); + Assert.assertNotNull(entity.getPrimaryFiltersJAXB()); + Assert.assertTrue(entity.getPrimaryFilters() instanceof HashMap); + Assert.assertTrue(entity.getPrimaryFiltersJAXB() instanceof HashMap); + Assert.assertEquals( + entity.getPrimaryFilters(), entity.getPrimaryFiltersJAXB()); + } + + private static void assertRelatedEntities(TimelineEntity entity) { + Assert.assertNotNull(entity.getRelatedEntities()); + Assert.assertNotNull(entity.getRelatedEntitiesJAXB()); + Assert.assertTrue(entity.getRelatedEntities() instanceof HashMap); + Assert.assertTrue(entity.getRelatedEntitiesJAXB() instanceof HashMap); + Assert.assertEquals( + entity.getRelatedEntities(), entity.getRelatedEntitiesJAXB()); + } + + private static void assertOtherInfo(TimelineEntity entity) { + Assert.assertNotNull(entity.getOtherInfo()); + Assert.assertNotNull(entity.getOtherInfoJAXB()); + Assert.assertTrue(entity.getOtherInfo() instanceof HashMap); + Assert.assertTrue(entity.getOtherInfoJAXB() instanceof HashMap); + Assert.assertEquals(entity.getOtherInfo(), entity.getOtherInfoJAXB()); + } + + private static void assertEventInfo(TimelineEvent event) { + Assert.assertNotNull(event); + Assert.assertNotNull(event.getEventInfoJAXB()); + Assert.assertTrue(event.getEventInfo() instanceof HashMap); + Assert.assertTrue(event.getEventInfoJAXB() instanceof HashMap); + Assert.assertEquals(event.getEventInfo(), event.getEventInfoJAXB()); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java index f29093085a..0907f2c154 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java @@ -42,7 +42,6 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import javax.ws.rs.core.Response.Status; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; @@ -53,11 +52,11 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain; +import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents; -import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain; -import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.timeline.EntityIdentifier; @@ -272,7 +271,7 @@ public TimelinePutResponse postEntities( @PUT @Path("/domain") @Consumes({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */}) - public Response putDomain( + public TimelinePutResponse putDomain( @Context HttpServletRequest req, @Context HttpServletResponse res, TimelineDomain domain) { @@ -295,7 +294,7 @@ public Response putDomain( throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR); } - return Response.status(Status.OK).build(); + return new TimelinePutResponse(); } /**