From fc8485d8934edf02bdef746d022d1125680e6efe Mon Sep 17 00:00:00 2001 From: Sangjin Lee Date: Mon, 4 May 2015 16:10:20 -0700 Subject: [PATCH] YARN-3551. Consolidate data model change according to the backend implementation (Zhijie Shen via sale) (cherry picked from commit 557a3950bddc837469244835f5577899080115d8) --- .../mapred/TimelineServicePerformanceV2.java | 2 +- .../timelineservice/TimelineEntity.java | 16 +-- .../timelineservice/TimelineMetric.java | 131 ++++++++---------- .../TestTimelineServiceRecords.java | 81 ++++++++--- .../monitor/ContainersMonitorImpl.java | 4 +- .../TestTimelineServiceClientIntegration.java | 6 + 6 files changed, 143 insertions(+), 97 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java index de4661792e..1c2e28df44 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java @@ -261,7 +261,7 @@ public void map(IntWritable key, IntWritable val, Context context) // add a metric TimelineMetric metric = new TimelineMetric(); metric.setId("foo_metric"); - metric.setSingleData(123456789L); + metric.addValue(System.currentTimeMillis(), 123456789L); entity.addMetric(metric); // add a config entity.addConfig("foo", "bar"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java index 6cab7531b8..3be7f52e4a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java @@ -80,7 +80,7 @@ public String toString() { private TimelineEntity real; private Identifier identifier; private HashMap info = new HashMap<>(); - private HashMap configs = new HashMap<>(); + private HashMap configs = new HashMap<>(); private Set metrics = new HashSet<>(); private Set events = new HashSet<>(); private HashMap> isRelatedToEntities = new HashMap<>(); @@ -213,7 +213,7 @@ public void addInfo(String key, Object value) { // required by JAXB @InterfaceAudience.Private @XmlElement(name = "configs") - public HashMap getConfigsJAXB() { + public HashMap getConfigsJAXB() { if (real == null) { return configs; } else { @@ -221,7 +221,7 @@ public HashMap getConfigsJAXB() { } } - public Map getConfigs() { + public Map getConfigs() { if (real == null) { return configs; } else { @@ -229,19 +229,19 @@ public Map getConfigs() { } } - public void setConfigs(Map configs) { + public void setConfigs(Map configs) { if (real == null) { if (configs != null && !(configs instanceof HashMap)) { - this.configs = new HashMap(configs); + this.configs = new HashMap(configs); } else { - this.configs = (HashMap) configs; + this.configs = (HashMap) configs; } } else { real.setConfigs(configs); } } - public void addConfigs(Map configs) { + public void addConfigs(Map configs) { if (real == null) { this.configs.putAll(configs); } else { @@ -249,7 +249,7 @@ public void addConfigs(Map configs) { } } - public void addConfig(String key, Object value) { + public void addConfig(String key, String value) { if (real == null) { configs.put(key, value); } else { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java index 57babbf633..c897d39f95 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java @@ -24,24 +24,47 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; -import java.util.HashMap; -import java.util.LinkedHashMap; +import java.util.Comparator; import java.util.Map; +import java.util.TreeMap; @XmlRootElement(name = "metric") @XmlAccessorType(XmlAccessType.NONE) @InterfaceAudience.Public @InterfaceStability.Unstable public class TimelineMetric { + + public static enum Type { + SINGLE_VALUE, + TIME_SERIES + } + + private Type type; private String id; - private HashMap info = new HashMap<>(); - private Object singleData; - private HashMap timeSeries = new LinkedHashMap<>(); - private long startTime; - private long endTime; + private Comparator reverseComparator = new Comparator() { + @Override + public int compare(Long l1, Long l2) { + return -l1.compareTo(l2); + } + }; + private TreeMap values = new TreeMap<>(reverseComparator); public TimelineMetric() { + this(Type.SINGLE_VALUE); + } + public TimelineMetric(Type type) { + this.type = type; + } + + + @XmlElement(name = "type") + public Type getType() { + return type; + } + + public void setType(Type type) { + this.type = type; } @XmlElement(name = "id") @@ -55,82 +78,50 @@ public void setId(String id) { // required by JAXB @InterfaceAudience.Private - @XmlElement(name = "info") - public HashMap getInfoJAXB() { - return info; + @XmlElement(name = "values") + public TreeMap getValuesJAXB() { + return values; } - public Map getInfo() { - return info; + public Map getValues() { + return values; } - public void setInfo(Map info) { - if (info != null && !(info instanceof HashMap)) { - this.info = new HashMap(info); + public void setValues(Map values) { + if (type == Type.SINGLE_VALUE) { + overwrite(values); } else { - this.info = (HashMap) info; + if (values != null) { + this.values = new TreeMap(reverseComparator); + this.values.putAll(values); + } else { + this.values = null; + } } } - public void addInfo(Map info) { - this.info.putAll(info); - } - - public void addInfo(String key, Object value) { - info.put(key, value); - } - - @XmlElement(name = "data") - public Object getSingleData() { - return singleData; - } - - public void setSingleData(Object singleData) { - this.singleData = singleData; - } - - // required by JAXB - @InterfaceAudience.Private - @XmlElement(name = "timeseries") - public HashMap getTimeSeriesJAXB() { - return timeSeries; - } - - public Map getTimeSeries() { - return timeSeries; - } - - public void setTimeSeries(Map timeSeries) { - if (timeSeries != null && !(timeSeries instanceof LinkedHashMap)) { - this.timeSeries = new LinkedHashMap(timeSeries); + public void addValues(Map values) { + if (type == Type.SINGLE_VALUE) { + overwrite(values); } else { - this.timeSeries = (LinkedHashMap) timeSeries; + this.values.putAll(values); } } - public void addTimeSeries(Map timeSeries) { - this.timeSeries.putAll(timeSeries); + public void addValue(long timestamp, Number value) { + if (type == Type.SINGLE_VALUE) { + values.clear(); + } + values.put(timestamp, value); } - public void addTimeSeriesData(long timestamp, Object value) { - timeSeries.put(timestamp, value); - } - - @XmlElement(name = "starttime") - public long getStartTime() { - return startTime; - } - - public void setStartTime(long startTime) { - this.startTime = startTime; - } - - @XmlElement(name = "endtime") - public long getEndTime() { - return endTime; - } - - public void setEndTime(long endTime) { - this.endTime = endTime; + private void overwrite(Map values) { + if (values.size() > 1) { + throw new IllegalArgumentException( + "Values cannot contain more than one point in " + + Type.SINGLE_VALUE + " mode"); + } + this.values.clear(); + this.values.putAll(values); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java index caa3f3f73e..95b922e0e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java @@ -28,7 +28,11 @@ import org.junit.Test; import org.junit.Assert; +import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; public class TestTimelineServiceRecords { @@ -41,38 +45,83 @@ public void testTimelineEntities() throws Exception { entity.setType("test type 1"); entity.setId("test id 1"); entity.addInfo("test info key 1", "test info value 1"); - entity.addInfo("test info key 2", "test info value 2"); + entity.addInfo("test info key 2", + Arrays.asList("test info value 2", "test info value 3")); + entity.addInfo("test info key 3", true); + Assert.assertTrue( + entity.getInfo().get("test info key 3") instanceof Boolean); entity.addConfig("test config key 1", "test config value 1"); entity.addConfig("test config key 2", "test config value 2"); - TimelineMetric metric1 = new TimelineMetric(); + + TimelineMetric metric1 = + new TimelineMetric(TimelineMetric.Type.TIME_SERIES); metric1.setId("test metric id 1"); - metric1.addInfo("test info key 1", "test info value 1"); - metric1.addInfo("test info key 2", "test info value 2"); - metric1.addTimeSeriesData(1L, "test time series 1"); - metric1.addTimeSeriesData(2L, "test time series 2"); - metric1.setStartTime(0L); - metric1.setEndTime(1L); + metric1.addValue(1L, 1.0F); + metric1.addValue(3L, 3.0D); + metric1.addValue(2L, 2); + Assert.assertEquals(TimelineMetric.Type.TIME_SERIES, metric1.getType()); + Iterator> itr = + metric1.getValues().entrySet().iterator(); + Map.Entry entry = itr.next(); + Assert.assertEquals(new Long(3L), entry.getKey()); + Assert.assertEquals(new Double(3.0D), entry.getValue()); + entry = itr.next(); + Assert.assertEquals(new Long(2L), entry.getKey()); + Assert.assertEquals(new Integer(2), entry.getValue()); + entry = itr.next(); + Assert.assertEquals(new Long(1L), entry.getKey()); + Assert.assertEquals(new Float(1.0F), entry.getValue()); + Assert.assertFalse(itr.hasNext()); entity.addMetric(metric1); - TimelineMetric metric2 = new TimelineMetric(); + + TimelineMetric metric2 = + new TimelineMetric(TimelineMetric.Type.SINGLE_VALUE); metric2.setId("test metric id 1"); - metric2.addInfo("test info key 1", "test info value 1"); - metric2.addInfo("test info key 2", "test info value 2"); - metric2.setSingleData("test info value 3"); - metric1.setStartTime(0L); - metric1.setEndTime(1L); + metric2.addValue(3L, (short) 3); + Assert.assertEquals(TimelineMetric.Type.SINGLE_VALUE, metric2.getType()); + Assert.assertTrue( + metric2.getValues().values().iterator().next() instanceof Short); + Map points = new HashMap<>(); + points.put(4L, 4.0D); + points.put(5L, 5.0D); + try { + metric2.setValues(points); + Assert.fail(); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains( + "Values cannot contain more than one point in")); + } + try { + metric2.addValues(points); + Assert.fail(); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains( + "Values cannot contain more than one point in")); + } entity.addMetric(metric2); + TimelineEvent event1 = new TimelineEvent(); event1.setId("test event id 1"); event1.addInfo("test info key 1", "test info value 1"); - event1.addInfo("test info key 2", "test info value 2"); + event1.addInfo("test info key 2", + Arrays.asList("test info value 2", "test info value 3")); + event1.addInfo("test info key 3", true); + Assert.assertTrue( + event1.getInfo().get("test info key 3") instanceof Boolean); event1.setTimestamp(0L); entity.addEvent(event1); + TimelineEvent event2 = new TimelineEvent(); event2.setId("test event id 2"); event2.addInfo("test info key 1", "test info value 1"); - event2.addInfo("test info key 2", "test info value 2"); + event2.addInfo("test info key 2", + Arrays.asList("test info value 2", "test info value 3")); + event2.addInfo("test info key 3", true); + Assert.assertTrue( + event2.getInfo().get("test info key 3") instanceof Boolean); event2.setTimestamp(1L); entity.addEvent(event2); + entity.setCreatedTime(0L); entity.setModifiedTime(1L); entity.addRelatesToEntity("test type 2", "test id 2"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java index d57c91fd38..c1fa72e285 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java @@ -572,7 +572,7 @@ public void run() { ResourceCalculatorProcessTree.UNAVAILABLE) { TimelineMetric memoryMetric = new TimelineMetric(); memoryMetric.setId(ContainerMetric.MEMORY.toString() + pId); - memoryMetric.addTimeSeriesData(currentTime, currentPmemUsage); + memoryMetric.addValue(currentTime, currentPmemUsage); entity.addMetric(memoryMetric); } // if cpuUsageTotalCoresPercentage data is available @@ -580,7 +580,7 @@ public void run() { ResourceCalculatorProcessTree.UNAVAILABLE) { TimelineMetric cpuMetric = new TimelineMetric(); cpuMetric.setId(ContainerMetric.CPU.toString() + pId); - cpuMetric.addTimeSeriesData(currentTime, + cpuMetric.addValue(currentTime, cpuUsageTotalCoresPercentage); entity.addMetric(cpuMetric); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java index 044148a4b6..69031a25d0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java @@ -81,6 +81,12 @@ public void testPutEntities() throws Exception { TimelineEntity entity = new TimelineEntity(); entity.setType("test entity type"); entity.setId("test entity id"); + TimelineMetric metric = + new TimelineMetric(TimelineMetric.Type.TIME_SERIES); + metric.setId("test metric id"); + metric.addValue(1L, 1.0D); + metric.addValue(2L, 2.0D); + entity.addMetric(metric); client.putEntities(entity); client.putEntitiesAsync(entity); } finally {