big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-3551. Consolidate data model change according to the backend implementation (Zhijie Shen via sale)

Browse Source 
(cherry picked from commit 557a3950bddc837469244835f5577899080115d8)
This commit is contained in:
Sangjin Lee 2015-05-04 16:10:20 -07:00
parent 8c7b6dd2c7
commit fc8485d893
6 changed files with 143 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java
View File

@ -261,7 +261,7 @@ public void map(IntWritable key, IntWritable val, Context context)
// add a metric
TimelineMetric metric = new TimelineMetric();
metric.setId("foo_metric");
metric.setSingleData(123456789L);
metric.addValue(System.currentTimeMillis(), 123456789L);
entity.addMetric(metric);
// add a config
entity.addConfig("foo", "bar");

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
View File

@ -80,7 +80,7 @@ public String toString() {
private TimelineEntity real;
private Identifier identifier;
private HashMap<String, Object> info = new HashMap<>();
private HashMap<String, Object> configs = new HashMap<>();
private HashMap<String, String> configs = new HashMap<>();
private Set<TimelineMetric> metrics = new HashSet<>();
private Set<TimelineEvent> events = new HashSet<>();
private HashMap<String, Set<String>> isRelatedToEntities = new HashMap<>();
@ -213,7 +213,7 @@ public void addInfo(String key, Object value) {
// required by JAXB
@InterfaceAudience.Private
@XmlElement(name = "configs")
public HashMap<String, Object> getConfigsJAXB() {
public HashMap<String, String> getConfigsJAXB() {
if (real == null) {
return configs;
} else {
@ -221,7 +221,7 @@ public HashMap<String, Object> getConfigsJAXB() {
}
}
public Map<String, Object> getConfigs() {
public Map<String, String> getConfigs() {
if (real == null) {
return configs;
} else {
@ -229,19 +229,19 @@ public Map<String, Object> getConfigs() {
}
}
public void setConfigs(Map<String, Object> configs) {
public void setConfigs(Map<String, String> configs) {
if (real == null) {
if (configs != null && !(configs instanceof HashMap)) {
this.configs = new HashMap<String, Object>(configs);
this.configs = new HashMap<String, String>(configs);
} else {
this.configs = (HashMap<String, Object>) configs;
this.configs = (HashMap<String, String>) configs;
}
} else {
real.setConfigs(configs);
}
}
public void addConfigs(Map<String, Object> configs) {
public void addConfigs(Map<String, String> configs) {
if (real == null) {
this.configs.putAll(configs);
} else {
@ -249,7 +249,7 @@ public void addConfigs(Map<String, Object> configs) {
}
}
public void addConfig(String key, Object value) {
public void addConfig(String key, String value) {
if (real == null) {
configs.put(key, value);
} else {

131
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
View File

@ -24,24 +24,47 @@
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Comparator;
import java.util.Map;
import java.util.TreeMap;
@XmlRootElement(name = "metric")
@XmlAccessorType(XmlAccessType.NONE)
@InterfaceAudience.Public
@InterfaceStability.Unstable
public class TimelineMetric {
public static enum Type {
SINGLE_VALUE,
TIME_SERIES
}
private Type type;
private String id;
private HashMap<String, Object> info = new HashMap<>();
private Object singleData;
private HashMap<Long, Object> timeSeries = new LinkedHashMap<>();
private long startTime;
private long endTime;
private Comparator<Long> reverseComparator = new Comparator<Long>() {
@Override
public int compare(Long l1, Long l2) {
return -l1.compareTo(l2);
}
};
private TreeMap<Long, Number> values = new TreeMap<>(reverseComparator);
public TimelineMetric() {
this(Type.SINGLE_VALUE);
}
public TimelineMetric(Type type) {
this.type = type;
}
@XmlElement(name = "type")
public Type getType() {
return type;
}
public void setType(Type type) {
this.type = type;
}
@XmlElement(name = "id")
@ -55,82 +78,50 @@ public void setId(String id) {
// required by JAXB
@InterfaceAudience.Private
@XmlElement(name = "info")
public HashMap<String, Object> getInfoJAXB() {
return info;
@XmlElement(name = "values")
public TreeMap<Long, Number> getValuesJAXB() {
return values;
}
public Map<String, Object> getInfo() {
return info;
public Map<Long, Number> getValues() {
return values;
}
public void setInfo(Map<String, Object> info) {
if (info != null && !(info instanceof HashMap)) {
this.info = new HashMap<String, Object>(info);
public void setValues(Map<Long, Number> values) {
if (type == Type.SINGLE_VALUE) {
overwrite(values);
} else {
this.info = (HashMap<String, Object>) info;
}
}
public void addInfo(Map<String, Object> info) {
this.info.putAll(info);
}
public void addInfo(String key, Object value) {
info.put(key, value);
}
@XmlElement(name = "data")
public Object getSingleData() {
return singleData;
}
public void setSingleData(Object singleData) {
this.singleData = singleData;
}
// required by JAXB
@InterfaceAudience.Private
@XmlElement(name = "timeseries")
public HashMap<Long, Object> getTimeSeriesJAXB() {
return timeSeries;
}
public Map<Long, Object> getTimeSeries() {
return timeSeries;
}
public void setTimeSeries(Map<Long, Object> timeSeries) {
if (timeSeries != null && !(timeSeries instanceof LinkedHashMap)) {
this.timeSeries = new LinkedHashMap<Long, Object>(timeSeries);
if (values != null) {
this.values = new TreeMap<Long, Number>(reverseComparator);
this.values.putAll(values);
} else {
this.timeSeries = (LinkedHashMap<Long, Object>) timeSeries;
this.values = null;
}
}
}
public void addTimeSeries(Map<Long, Object> timeSeries) {
this.timeSeries.putAll(timeSeries);
public void addValues(Map<Long, Number> values) {
if (type == Type.SINGLE_VALUE) {
overwrite(values);
} else {
this.values.putAll(values);
}
}
public void addTimeSeriesData(long timestamp, Object value) {
timeSeries.put(timestamp, value);
public void addValue(long timestamp, Number value) {
if (type == Type.SINGLE_VALUE) {
values.clear();
}
values.put(timestamp, value);
}
@XmlElement(name = "starttime")
public long getStartTime() {
return startTime;
private void overwrite(Map<Long, Number> values) {
if (values.size() > 1) {
throw new IllegalArgumentException(
"Values cannot contain more than one point in " +
Type.SINGLE_VALUE + " mode");
}
public void setStartTime(long startTime) {
this.startTime = startTime;
}
@XmlElement(name = "endtime")
public long getEndTime() {
return endTime;
}
public void setEndTime(long endTime) {
this.endTime = endTime;
this.values.clear();
this.values.putAll(values);
}
}

81
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
View File

@ -28,7 +28,11 @@
import org.junit.Test;
import org.junit.Assert;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
public class TestTimelineServiceRecords {
@ -41,38 +45,83 @@ public void testTimelineEntities() throws Exception {
entity.setType("test type 1");
entity.setId("test id 1");
entity.addInfo("test info key 1", "test info value 1");
entity.addInfo("test info key 2", "test info value 2");
entity.addInfo("test info key 2",
Arrays.asList("test info value 2", "test info value 3"));
entity.addInfo("test info key 3", true);
Assert.assertTrue(
entity.getInfo().get("test info key 3") instanceof Boolean);
entity.addConfig("test config key 1", "test config value 1");
entity.addConfig("test config key 2", "test config value 2");
TimelineMetric metric1 = new TimelineMetric();
TimelineMetric metric1 =
new TimelineMetric(TimelineMetric.Type.TIME_SERIES);
metric1.setId("test metric id 1");
metric1.addInfo("test info key 1", "test info value 1");
metric1.addInfo("test info key 2", "test info value 2");
metric1.addTimeSeriesData(1L, "test time series 1");
metric1.addTimeSeriesData(2L, "test time series 2");
metric1.setStartTime(0L);
metric1.setEndTime(1L);
metric1.addValue(1L, 1.0F);
metric1.addValue(3L, 3.0D);
metric1.addValue(2L, 2);
Assert.assertEquals(TimelineMetric.Type.TIME_SERIES, metric1.getType());
Iterator<Map.Entry<Long, Number>> itr =
metric1.getValues().entrySet().iterator();
Map.Entry<Long, Number> entry = itr.next();
Assert.assertEquals(new Long(3L), entry.getKey());
Assert.assertEquals(new Double(3.0D), entry.getValue());
entry = itr.next();
Assert.assertEquals(new Long(2L), entry.getKey());
Assert.assertEquals(new Integer(2), entry.getValue());
entry = itr.next();
Assert.assertEquals(new Long(1L), entry.getKey());
Assert.assertEquals(new Float(1.0F), entry.getValue());
Assert.assertFalse(itr.hasNext());
entity.addMetric(metric1);
TimelineMetric metric2 = new TimelineMetric();
TimelineMetric metric2 =
new TimelineMetric(TimelineMetric.Type.SINGLE_VALUE);
metric2.setId("test metric id 1");
metric2.addInfo("test info key 1", "test info value 1");
metric2.addInfo("test info key 2", "test info value 2");
metric2.setSingleData("test info value 3");
metric1.setStartTime(0L);
metric1.setEndTime(1L);
metric2.addValue(3L, (short) 3);
Assert.assertEquals(TimelineMetric.Type.SINGLE_VALUE, metric2.getType());
Assert.assertTrue(
metric2.getValues().values().iterator().next() instanceof Short);
Map<Long, Number> points = new HashMap<>();
points.put(4L, 4.0D);
points.put(5L, 5.0D);
try {
metric2.setValues(points);
Assert.fail();
} catch (IllegalArgumentException e) {
Assert.assertTrue(e.getMessage().contains(
"Values cannot contain more than one point in"));
}
try {
metric2.addValues(points);
Assert.fail();
} catch (IllegalArgumentException e) {
Assert.assertTrue(e.getMessage().contains(
"Values cannot contain more than one point in"));
}
entity.addMetric(metric2);
TimelineEvent event1 = new TimelineEvent();
event1.setId("test event id 1");
event1.addInfo("test info key 1", "test info value 1");
event1.addInfo("test info key 2", "test info value 2");
event1.addInfo("test info key 2",
Arrays.asList("test info value 2", "test info value 3"));
event1.addInfo("test info key 3", true);
Assert.assertTrue(
event1.getInfo().get("test info key 3") instanceof Boolean);
event1.setTimestamp(0L);
entity.addEvent(event1);
TimelineEvent event2 = new TimelineEvent();
event2.setId("test event id 2");
event2.addInfo("test info key 1", "test info value 1");
event2.addInfo("test info key 2", "test info value 2");
event2.addInfo("test info key 2",
Arrays.asList("test info value 2", "test info value 3"));
event2.addInfo("test info key 3", true);
Assert.assertTrue(
event2.getInfo().get("test info key 3") instanceof Boolean);
event2.setTimestamp(1L);
entity.addEvent(event2);
entity.setCreatedTime(0L);
entity.setModifiedTime(1L);
entity.addRelatesToEntity("test type 2", "test id 2");

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
View File

@ -572,7 +572,7 @@ public void run() {
ResourceCalculatorProcessTree.UNAVAILABLE) {
TimelineMetric memoryMetric = new TimelineMetric();
memoryMetric.setId(ContainerMetric.MEMORY.toString() + pId);
memoryMetric.addTimeSeriesData(currentTime, currentPmemUsage);
memoryMetric.addValue(currentTime, currentPmemUsage);
entity.addMetric(memoryMetric);
}
// if cpuUsageTotalCoresPercentage data is available
@ -580,7 +580,7 @@ public void run() {
ResourceCalculatorProcessTree.UNAVAILABLE) {
TimelineMetric cpuMetric = new TimelineMetric();
cpuMetric.setId(ContainerMetric.CPU.toString() + pId);
cpuMetric.addTimeSeriesData(currentTime,
cpuMetric.addValue(currentTime,
cpuUsageTotalCoresPercentage);
entity.addMetric(cpuMetric);
}

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
View File

@ -81,6 +81,12 @@ public void testPutEntities() throws Exception {
TimelineEntity entity = new TimelineEntity();
entity.setType("test entity type");
entity.setId("test entity id");
TimelineMetric metric =
new TimelineMetric(TimelineMetric.Type.TIME_SERIES);
metric.setId("test metric id");
metric.addValue(1L, 1.0D);
metric.addValue(2L, 2.0D);
entity.addMetric(metric);
client.putEntities(entity);
client.putEntitiesAsync(entity);
} finally {