big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-6720. Inconsistent values of counters across tasks and job reported to timeline service. Contributed by Varun Saxena

Browse Source
This commit is contained in:
Naganarasimha 2016-06-22 23:32:19 +05:30 committed by Sangjin Lee
parent 6d943038f6
commit 8bf87eede2
3 changed files with 16 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
View File

@ -1125,6 +1125,10 @@ private void processEventForTimelineServer(HistoryEvent event, JobId jobId,
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
createBaseEntity(event, timestamp, entityType, setCreatedTime); createBaseEntity(event, timestamp, entityType, setCreatedTime);
entity.setId(taskId); entity.setId(taskId);
if (event.getEventType() == EventType.TASK_STARTED) {
entity.addInfo("TASK_TYPE",
((TaskStartedEvent)event).getTaskType().toString());
}
entity.addIsRelatedToEntity(relatedJobEntity, jobId.toString()); entity.addIsRelatedToEntity(relatedJobEntity, jobId.toString());
return entity; return entity;
} }

12
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java
View File

@ -158,12 +158,12 @@ public TimelineEvent toTimelineEvent() {
@Override @Override
public Set<TimelineMetric> getTimelineMetrics() { public Set<TimelineMetric> getTimelineMetrics() {
Set<TimelineMetric> jobMetrics = JobHistoryEventUtils Set<TimelineMetric> jobMetrics = JobHistoryEventUtils.
.countersToTimelineMetric(getMapCounters(), finishTime); countersToTimelineMetric(getTotalCounters(), finishTime);
jobMetrics.addAll(JobHistoryEventUtils jobMetrics.addAll(JobHistoryEventUtils.
.countersToTimelineMetric(getReduceCounters(), finishTime)); countersToTimelineMetric(getMapCounters(), finishTime, "MAP:"));
jobMetrics.addAll(JobHistoryEventUtils jobMetrics.addAll(JobHistoryEventUtils.
.countersToTimelineMetric(getTotalCounters(), finishTime)); countersToTimelineMetric(getReduceCounters(), finishTime, "REDUCE:"));
return jobMetrics; return jobMetrics;
} }
} }

7
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java
View File

@ -61,11 +61,16 @@ public static JsonNode countersToJSON(Counters counters) {
public static Set<TimelineMetric> countersToTimelineMetric(Counters counters, public static Set<TimelineMetric> countersToTimelineMetric(Counters counters,
long timestamp) { long timestamp) {
return countersToTimelineMetric(counters, timestamp, "");
}
public static Set<TimelineMetric> countersToTimelineMetric(Counters counters,
long timestamp, String groupNamePrefix) {
Set<TimelineMetric> entityMetrics = new HashSet<TimelineMetric>(); Set<TimelineMetric> entityMetrics = new HashSet<TimelineMetric>();
for (CounterGroup g : counters) { for (CounterGroup g : counters) {
String groupName = g.getName(); String groupName = g.getName();
for (Counter c : g) { for (Counter c : g) {
String name = groupName + ":" + c.getName(); String name = groupNamePrefix + groupName + ":" + c.getName();
TimelineMetric metric = new TimelineMetric(); TimelineMetric metric = new TimelineMetric();
metric.setId(name); metric.setId(name);
metric.addValue(timestamp, c.getValue()); metric.addValue(timestamp, c.getValue());