YARN-2804. Fixed Timeline service to not fill the logs with JAXB bindings exceptions. Contributed by Zhijie Shen.

hadoop-yarn-project/CHANGES.txt

@@ -859,6 +859,9 @@ Release 2.6.0 - UNRELEASED
     YARN-2010. Handle app-recovery failures gracefully. 
     (Jian He and Karthik Kambatla via kasha)
 
+    YARN-2804. Fixed Timeline service to not fill the logs with JAXB bindings
+    exceptions. (Zhijie Shen via vinodkv)
+
 Release 2.5.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEntity.java

@@ -31,6 +31,7 @@
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlRootElement;
 
+import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 
@@ -58,11 +59,11 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
   private String entityId;
   private Long startTime;
   private List<TimelineEvent> events = new ArrayList<TimelineEvent>();
-  private Map<String, Set<String>> relatedEntities =
+  private HashMap<String, Set<String>> relatedEntities =
       new HashMap<String, Set<String>>();
-  private Map<String, Set<Object>> primaryFilters =
+  private HashMap<String, Set<Object>> primaryFilters =
       new HashMap<String, Set<Object>>();
-  private Map<String, Object> otherInfo =
+  private HashMap<String, Object> otherInfo =
       new HashMap<String, Object>();
   private String domainId;
 
@@ -175,11 +176,17 @@ public void setEvents(List<TimelineEvent> events) {
    * 
    * @return the related entities
    */
-  @XmlElement(name = "relatedentities")
   public Map<String, Set<String>> getRelatedEntities() {
     return relatedEntities;
   }
 
+  // Required by JAXB
+  @Private
+  @XmlElement(name = "relatedentities")
+  public HashMap<String, Set<String>> getRelatedEntitiesJAXB() {
+    return relatedEntities;
+  }
+
   /**
    * Add an entity to the existing related entity map
    * 
@@ -224,7 +231,11 @@ public void addRelatedEntities(Map<String, Set<String>> relatedEntities) {
    */
   public void setRelatedEntities(
       Map<String, Set<String>> relatedEntities) {
-    this.relatedEntities = relatedEntities;
+    if (relatedEntities != null && !(relatedEntities instanceof HashMap)) {
+      this.relatedEntities = new HashMap<String, Set<String>>(relatedEntities);
+    } else {
+      this.relatedEntities = (HashMap<String, Set<String>>) relatedEntities;
+    }
   }
 
   /**
@@ -232,11 +243,17 @@ public void setRelatedEntities(
    * 
    * @return the primary filters
    */
-  @XmlElement(name = "primaryfilters")
   public Map<String, Set<Object>> getPrimaryFilters() {
     return primaryFilters;
   }
 
+  // Required by JAXB
+  @Private
+  @XmlElement(name = "primaryfilters")
+  public HashMap<String, Set<Object>> getPrimaryFiltersJAXB() {
+    return primaryFilters;
+  }
+
   /**
    * Add a single piece of primary filter to the existing primary filter map
    * 
@@ -280,7 +297,11 @@ public void addPrimaryFilters(Map<String, Set<Object>> primaryFilters) {
    *          a map of primary filters
    */
   public void setPrimaryFilters(Map<String, Set<Object>> primaryFilters) {
-    this.primaryFilters = primaryFilters;
+    if (primaryFilters != null && !(primaryFilters instanceof HashMap)) {
+      this.primaryFilters = new HashMap<String, Set<Object>>(primaryFilters);
+    } else {
+      this.primaryFilters = (HashMap<String, Set<Object>>) primaryFilters;
+    }
   }
 
   /**
@@ -288,11 +309,17 @@ public void setPrimaryFilters(Map<String, Set<Object>> primaryFilters) {
    * 
    * @return the other information of the entity
    */
-  @XmlElement(name = "otherinfo")
   public Map<String, Object> getOtherInfo() {
     return otherInfo;
   }
 
+  // Required by JAXB
+  @Private
+  @XmlElement(name = "otherinfo")
+  public HashMap<String, Object> getOtherInfoJAXB() {
+    return otherInfo;
+  }
+
   /**
    * Add one piece of other information of the entity to the existing other info
    * map
@@ -323,7 +350,11 @@ public void addOtherInfo(Map<String, Object> otherInfo) {
    *          a map of other information
    */
   public void setOtherInfo(Map<String, Object> otherInfo) {
-    this.otherInfo = otherInfo;
+    if (otherInfo != null && !(otherInfo instanceof HashMap)) {
+      this.otherInfo = new HashMap<String, Object>(otherInfo);
+    } else {
+      this.otherInfo = (HashMap<String, Object>) otherInfo;
+    }
   }
 
   /**

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEvent.java

@@ -26,6 +26,7 @@
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlRootElement;
 
+import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 
@@ -43,7 +44,7 @@ public class TimelineEvent implements Comparable<TimelineEvent> {
 
   private long timestamp;
   private String eventType;
-  private Map<String, Object> eventInfo = new HashMap<String, Object>();
+  private HashMap<String, Object> eventInfo = new HashMap<String, Object>();
 
   public TimelineEvent() {
   }
@@ -93,11 +94,17 @@ public void setEventType(String eventType) {
    * 
    * @return the information of the event
    */
-  @XmlElement(name = "eventinfo")
   public Map<String, Object> getEventInfo() {
     return eventInfo;
   }
 
+  // Required by JAXB
+  @Private
+  @XmlElement(name = "eventinfo")
+  public HashMap<String, Object> getEventInfoJAXB() {
+    return eventInfo;
+  }
+
   /**
    * Add one piece of the information of the event to the existing information
    * map
@@ -128,7 +135,11 @@ public void addEventInfo(Map<String, Object> eventInfo) {
    *          a map of of the information of the event
    */
   public void setEventInfo(Map<String, Object> eventInfo) {
-    this.eventInfo = eventInfo;
+    if (eventInfo != null && !(eventInfo instanceof HashMap)) {
+      this.eventInfo = new HashMap<String, Object>(eventInfo);
+    } else {
+      this.eventInfo = (HashMap<String, Object>) eventInfo;
+    }
   }
 
   @Override

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/TestTimelineRecords.java

@@ -19,19 +19,19 @@
 package org.apache.hadoop.yarn.api.records.timeline;
 
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
-
+import java.util.Map;
-import org.junit.Assert;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.WeakHashMap;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents;
-import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
 import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError;
 import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+import org.junit.Assert;
 import org.junit.Test;
 
 public class TestTimelineRecords {
@@ -202,4 +202,118 @@ public void testTimelineDomain() throws Exception {
     }
   }
 
+  @Test
+  public void testMapInterfaceOrTimelineRecords() throws Exception {
+    TimelineEntity entity = new TimelineEntity();
+    List<Map<String, Set<Object>>> primaryFiltersList =
+        new ArrayList<Map<String, Set<Object>>>();
+    primaryFiltersList.add(
+        Collections.singletonMap("pkey", Collections.singleton((Object) "pval")));
+    Map<String, Set<Object>> primaryFilters = new TreeMap<String, Set<Object>>();
+    primaryFilters.put("pkey1", Collections.singleton((Object) "pval1"));
+    primaryFilters.put("pkey2", Collections.singleton((Object) "pval2"));
+    primaryFiltersList.add(primaryFilters);
+    entity.setPrimaryFilters(null);
+    for (Map<String, Set<Object>> primaryFiltersToSet : primaryFiltersList) {
+      entity.setPrimaryFilters(primaryFiltersToSet);
+      assertPrimaryFilters(entity);
+
+      Map<String, Set<Object>> primaryFiltersToAdd =
+          new WeakHashMap<String, Set<Object>>();
+      primaryFiltersToAdd.put("pkey3", Collections.singleton((Object) "pval3"));
+      entity.addPrimaryFilters(primaryFiltersToAdd);
+      assertPrimaryFilters(entity);
+    }
+
+    List<Map<String, Set<String>>> relatedEntitiesList =
+        new ArrayList<Map<String, Set<String>>>();
+    relatedEntitiesList.add(
+        Collections.singletonMap("rkey", Collections.singleton("rval")));
+    Map<String, Set<String>> relatedEntities = new TreeMap<String, Set<String>>();
+    relatedEntities.put("rkey1", Collections.singleton("rval1"));
+    relatedEntities.put("rkey2", Collections.singleton("rval2"));
+    relatedEntitiesList.add(relatedEntities);
+    entity.setRelatedEntities(null);
+    for (Map<String, Set<String>> relatedEntitiesToSet : relatedEntitiesList) {
+      entity.setRelatedEntities(relatedEntitiesToSet);
+      assertRelatedEntities(entity);
+
+      Map<String, Set<String>> relatedEntitiesToAdd =
+          new WeakHashMap<String, Set<String>>();
+      relatedEntitiesToAdd.put("rkey3", Collections.singleton("rval3"));
+      entity.addRelatedEntities(relatedEntitiesToAdd);
+      assertRelatedEntities(entity);
+    }
+
+    List<Map<String, Object>> otherInfoList =
+        new ArrayList<Map<String, Object>>();
+    otherInfoList.add(Collections.singletonMap("okey", (Object) "oval"));
+    Map<String, Object> otherInfo = new TreeMap<String, Object>();
+    otherInfo.put("okey1", "oval1");
+    otherInfo.put("okey2", "oval2");
+    otherInfoList.add(otherInfo);
+    entity.setOtherInfo(null);
+    for (Map<String, Object> otherInfoToSet : otherInfoList) {
+      entity.setOtherInfo(otherInfoToSet);
+      assertOtherInfo(entity);
+
+      Map<String, Object> otherInfoToAdd = new WeakHashMap<String, Object>();
+      otherInfoToAdd.put("okey3", "oval3");
+      entity.addOtherInfo(otherInfoToAdd);
+      assertOtherInfo(entity);
+    }
+
+    TimelineEvent event = new TimelineEvent();
+    List<Map<String, Object>> eventInfoList =
+        new ArrayList<Map<String, Object>>();
+    eventInfoList.add(Collections.singletonMap("ekey", (Object) "eval"));
+    Map<String, Object> eventInfo = new TreeMap<String, Object>();
+    eventInfo.put("ekey1", "eval1");
+    eventInfo.put("ekey2", "eval2");
+    eventInfoList.add(eventInfo);
+    event.setEventInfo(null);
+    for (Map<String, Object> eventInfoToSet : eventInfoList) {
+      event.setEventInfo(eventInfoToSet);
+      assertEventInfo(event);
+
+      Map<String, Object> eventInfoToAdd = new WeakHashMap<String, Object>();
+      eventInfoToAdd.put("ekey3", "eval3");
+      event.addEventInfo(eventInfoToAdd);
+      assertEventInfo(event);
+    }
+  }
+
+  private static void assertPrimaryFilters(TimelineEntity entity) {
+    Assert.assertNotNull(entity.getPrimaryFilters());
+    Assert.assertNotNull(entity.getPrimaryFiltersJAXB());
+    Assert.assertTrue(entity.getPrimaryFilters() instanceof HashMap);
+    Assert.assertTrue(entity.getPrimaryFiltersJAXB() instanceof HashMap);
+    Assert.assertEquals(
+        entity.getPrimaryFilters(), entity.getPrimaryFiltersJAXB());
+  }
+
+  private static void assertRelatedEntities(TimelineEntity entity) {
+    Assert.assertNotNull(entity.getRelatedEntities());
+    Assert.assertNotNull(entity.getRelatedEntitiesJAXB());
+    Assert.assertTrue(entity.getRelatedEntities() instanceof HashMap);
+    Assert.assertTrue(entity.getRelatedEntitiesJAXB() instanceof HashMap);
+    Assert.assertEquals(
+        entity.getRelatedEntities(), entity.getRelatedEntitiesJAXB());
+  }
+
+  private static void assertOtherInfo(TimelineEntity entity) {
+    Assert.assertNotNull(entity.getOtherInfo());
+    Assert.assertNotNull(entity.getOtherInfoJAXB());
+    Assert.assertTrue(entity.getOtherInfo() instanceof HashMap);
+    Assert.assertTrue(entity.getOtherInfoJAXB() instanceof HashMap);
+    Assert.assertEquals(entity.getOtherInfo(), entity.getOtherInfoJAXB());
+  }
+
+  private static void assertEventInfo(TimelineEvent event) {
+    Assert.assertNotNull(event);
+    Assert.assertNotNull(event.getEventInfoJAXB());
+    Assert.assertTrue(event.getEventInfo() instanceof HashMap);
+    Assert.assertTrue(event.getEventInfoJAXB() instanceof HashMap);
+    Assert.assertEquals(event.getEventInfo(), event.getEventInfoJAXB());
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java

@@ -42,7 +42,6 @@
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
-import javax.ws.rs.core.Response.Status;
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
 import javax.xml.bind.annotation.XmlElement;
@@ -53,11 +52,11 @@
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains;
 import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.timeline.EntityIdentifier;
@@ -272,7 +271,7 @@ public TimelinePutResponse postEntities(
   @PUT
   @Path("/domain")
   @Consumes({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
-  public Response putDomain(
+  public TimelinePutResponse putDomain(
       @Context HttpServletRequest req,
       @Context HttpServletResponse res,
       TimelineDomain domain) {
@@ -295,7 +294,7 @@ public Response putDomain(
       throw new WebApplicationException(e,
           Response.Status.INTERNAL_SERVER_ERROR);
     }
-    return Response.status(Status.OK).build();
+    return new TimelinePutResponse();
   }
 
   /**