From 1ff6833bbacf5c4eeaff5e70553ac083a691bb21 Mon Sep 17 00:00:00 2001 From: Varun Saxena Date: Wed, 15 Jun 2016 22:10:56 +0530 Subject: [PATCH] YARN-5243. fix several rebase and other miscellaneous issues before merge. (Sangjin Lee via Varun Saxena) --- .../jobhistory/JobHistoryEventHandler.java | 42 +++++++++---------- .../v2/app/rm/RMContainerAllocator.java | 3 +- .../mapreduce/jobhistory/TestEvents.java | 2 +- .../mapreduce/jobhistory/AMStartedEvent.java | 2 +- .../mapreduce/jobhistory/HistoryEvent.java | 2 +- .../jobhistory/JobFinishedEvent.java | 2 +- .../jobhistory/JobInfoChangeEvent.java | 2 +- .../mapreduce/jobhistory/JobInitedEvent.java | 2 +- .../jobhistory/JobPriorityChangeEvent.java | 2 +- .../jobhistory/JobQueueChangeEvent.java | 2 +- .../jobhistory/JobStatusChangedEvent.java | 2 +- .../jobhistory/JobSubmittedEvent.java | 2 +- .../JobUnsuccessfulCompletionEvent.java | 2 +- .../jobhistory/MapAttemptFinishedEvent.java | 2 +- .../jobhistory/NormalizedResourceEvent.java | 2 +- .../ReduceAttemptFinishedEvent.java | 2 +- .../jobhistory/TaskAttemptFinishedEvent.java | 2 +- .../jobhistory/TaskAttemptStartedEvent.java | 2 +- ...askAttemptUnsuccessfulCompletionEvent.java | 2 +- .../mapreduce/jobhistory/TaskFailedEvent.java | 2 +- .../jobhistory/TaskFinishedEvent.java | 2 +- .../jobhistory/TaskStartedEvent.java | 2 +- .../jobhistory/TaskUpdatedEvent.java | 2 +- .../mapred/TestMRTimelineEventHandling.java | 28 ++++++------- .../apache/hadoop/test/MapredTestDriver.java | 2 +- .../dev-support/findbugs-exclude.xml | 5 --- .../distributedshell/ApplicationMaster.java | 2 +- .../TestDistributedShell.java | 2 +- .../client/api/impl/TimelineClientImpl.java | 4 +- .../src/main/resources/yarn-default.xml | 2 +- .../yarn/util/TestTimelineServiceHelper.java | 20 ++++----- .../impl/pb/NodeHeartbeatRequestPBImpl.java | 4 +- .../yarn/server/nodemanager/Context.java | 2 +- .../yarn/server/nodemanager/NodeManager.java | 13 ++++-- .../application/ApplicationImpl.java | 2 +- .../monitor/ContainersMonitorImpl.java | 6 +-- .../nodemanager/webapp/TestNMWebServer.java | 2 - .../pom.xml | 2 +- .../RMActiveServiceContext.java | 32 +------------- .../server/resourcemanager/RMContext.java | 4 +- .../server/resourcemanager/RMContextImpl.java | 2 +- .../resourcemanager/ResourceManager.java | 34 +++++---------- .../server/resourcemanager/rmapp/RMApp.java | 14 +++++-- .../resourcemanager/rmapp/RMAppImpl.java | 8 ++++ 44 files changed, 123 insertions(+), 154 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java index 887533d7fd..a09c17b08c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java @@ -137,7 +137,7 @@ public class JobHistoryEventHandler extends AbstractService protected volatile boolean forceJobCompletion = false; protected TimelineClient timelineClient; - + private boolean timelineServiceV2Enabled = false; private static String MAPREDUCE_JOB_ENTITY_TYPE = "MAPREDUCE_JOB"; @@ -263,16 +263,16 @@ protected void serviceInit(Configuration conf) throws Exception { MRJobConfig.MR_AM_HISTORY_USE_BATCHED_FLUSH_QUEUE_SIZE_THRESHOLD, MRJobConfig.DEFAULT_MR_AM_HISTORY_USE_BATCHED_FLUSH_QUEUE_SIZE_THRESHOLD); - // TODO replace MR specific configurations on timeline service with getting - // configuration from RM through registerApplicationMaster() in - // ApplicationMasterProtocol with return value for timeline service + // TODO replace MR specific configurations on timeline service with getting + // configuration from RM through registerApplicationMaster() in + // ApplicationMasterProtocol with return value for timeline service // configuration status: off, on_with_v1 or on_with_v2. if (conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, MRJobConfig.DEFAULT_MAPREDUCE_JOB_EMIT_TIMELINE_DATA)) { LOG.info("Emitting job history data to the timeline service is enabled"); if (YarnConfiguration.timelineServiceEnabled(conf)) { - timelineClient = + timelineClient = ((MRAppMaster.RunningAppContext)context).getTimelineClient(); timelineClient.init(conf); timelineServiceV2Enabled = @@ -1062,11 +1062,11 @@ private void processEventForTimelineServer(HistoryEvent event, JobId jobId, + "Server", ex); } } - - // create JobEntity from HistoryEvent with adding other info, like: + + // create JobEntity from HistoryEvent with adding other info, like: // jobId, timestamp and entityType. - private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity - createJobEntity(HistoryEvent event, long timestamp, JobId jobId, + private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity + createJobEntity(HistoryEvent event, long timestamp, JobId jobId, String entityType, boolean setCreatedTime) { org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = @@ -1093,16 +1093,16 @@ private void processEventForTimelineServer(HistoryEvent event, JobId jobId, return entity; } - // create BaseEntity from HistoryEvent with adding other info, like: + // create BaseEntity from HistoryEvent with adding other info, like: // timestamp and entityType. - private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity + private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity createBaseEntity(HistoryEvent event, long timestamp, String entityType, boolean setCreatedTime) { org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent tEvent = event.toTimelineEvent(); tEvent.setTimestamp(timestamp); - - org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = + + org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity(); entity.addEvent(tEvent); entity.setType(entityType); @@ -1115,10 +1115,10 @@ private void processEventForTimelineServer(HistoryEvent event, JobId jobId, } return entity; } - - // create TaskEntity from HistoryEvent with adding other info, like: + + // create TaskEntity from HistoryEvent with adding other info, like: // taskId, jobId, timestamp, entityType and relatedJobEntity. - private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity + private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity createTaskEntity(HistoryEvent event, long timestamp, String taskId, String entityType, String relatedJobEntity, JobId jobId, boolean setCreatedTime) { @@ -1128,12 +1128,12 @@ private void processEventForTimelineServer(HistoryEvent event, JobId jobId, entity.addIsRelatedToEntity(relatedJobEntity, jobId.toString()); return entity; } - - // create TaskAttemptEntity from HistoryEvent with adding other info, like: + + // create TaskAttemptEntity from HistoryEvent with adding other info, like: // timestamp, taskAttemptId, entityType, relatedTaskEntity and taskId. - private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity - createTaskAttemptEntity(HistoryEvent event, long timestamp, - String taskAttemptId, String entityType, String relatedTaskEntity, + private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity + createTaskAttemptEntity(HistoryEvent event, long timestamp, + String taskAttemptId, String entityType, String relatedTaskEntity, String taskId, boolean setCreatedTime) { org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = createBaseEntity(event, timestamp, entityType, setCreatedTime); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java index e1fc84fd20..ea2046b319 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java @@ -822,8 +822,9 @@ private List getResources() throws Exception { handleUpdatedNodes(response); handleJobPriorityChange(response); + // handle receiving the timeline collector address for this app String collectorAddr = response.getCollectorAddr(); - MRAppMaster.RunningAppContext appContext = + MRAppMaster.RunningAppContext appContext = (MRAppMaster.RunningAppContext)this.getContext(); if (collectorAddr != null && !collectorAddr.isEmpty() && appContext.getTimelineClient() != null) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java index f5f4a3ca37..ac510b3943 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java @@ -418,7 +418,7 @@ public TimelineEvent toTimelineEvent() { public Set getTimelineMetrics() { return null; } - + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java index 82db9f3e6f..a1447c58b4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java @@ -170,7 +170,7 @@ public long getSubmitTime() { public EventType getEventType() { return EventType.AM_STARTED; } - + @Override public TimelineEvent toTimelineEvent() { TimelineEvent tEvent = new TimelineEvent(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryEvent.java index cdafe46a64..1d59ebef89 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryEvent.java @@ -41,7 +41,7 @@ public interface HistoryEvent { /** Set the Avro datum wrapped by this. */ void setDatum(Object datum); - + /** Map HistoryEvent to TimelineEvent */ TimelineEvent toTimelineEvent(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java index 2efbc57935..76d689715e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java @@ -139,7 +139,7 @@ public Counters getMapCounters() { public Counters getReduceCounters() { return reduceCounters; } - + @Override public TimelineEvent toTimelineEvent() { TimelineEvent tEvent = new TimelineEvent(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInfoChangeEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInfoChangeEvent.java index eb9ad0d326..f5941aa044 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInfoChangeEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInfoChangeEvent.java @@ -66,7 +66,7 @@ public void setDatum(Object datum) { public EventType getEventType() { return EventType.JOB_INFO_CHANGED; } - + @Override public TimelineEvent toTimelineEvent() { TimelineEvent tEvent = new TimelineEvent(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java index d91208cdc3..784267f2b0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java @@ -77,7 +77,7 @@ public EventType getEventType() { } /** Get whether the job's map and reduce stages were combined */ public boolean getUberized() { return datum.getUberized(); } - + @Override public TimelineEvent toTimelineEvent() { TimelineEvent tEvent = new TimelineEvent(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobPriorityChangeEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobPriorityChangeEvent.java index ac4943e666..1616dd1962 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobPriorityChangeEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobPriorityChangeEvent.java @@ -66,7 +66,7 @@ public JobPriority getPriority() { public EventType getEventType() { return EventType.JOB_PRIORITY_CHANGED; } - + @Override public TimelineEvent toTimelineEvent() { TimelineEvent tEvent = new TimelineEvent(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java index b12006b53b..66f3781239 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java @@ -64,7 +64,7 @@ public String getJobQueueName() { } return null; } - + @Override public TimelineEvent toTimelineEvent() { TimelineEvent tEvent = new TimelineEvent(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobStatusChangedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobStatusChangedEvent.java index 0b33b4f580..0963b45b49 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobStatusChangedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobStatusChangedEvent.java @@ -62,7 +62,7 @@ public void setDatum(Object datum) { public EventType getEventType() { return EventType.JOB_STATUS_CHANGED; } - + @Override public TimelineEvent toTimelineEvent() { TimelineEvent tEvent = new TimelineEvent(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java index 7d05571fff..e394f5bbf6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java @@ -259,7 +259,7 @@ public TimelineEvent toTimelineEvent() { tEvent.addInfo("WORKFLOW_ADJACENCIES", getWorkflowAdjacencies()); tEvent.addInfo("WORKFLOW_TAGS", getWorkflowTags()); - + return tEvent; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java index 5d3c45faaf..ce6fa328ef 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java @@ -123,7 +123,7 @@ public String getDiagnostics() { final CharSequence diagnostics = datum.getDiagnostics(); return diagnostics == null ? NODIAGS : diagnostics.toString(); } - + @Override public TimelineEvent toTimelineEvent() { TimelineEvent tEvent = new TimelineEvent(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java index 1d28335449..3121c4e001 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java @@ -247,5 +247,5 @@ public Set getTimelineMetrics() { .countersToTimelineMetric(getCounters(), finishTime); return metrics; } - + } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java index d90ff93728..7adae23459 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java @@ -76,7 +76,7 @@ public Object getDatum() { public void setDatum(Object datum) { throw new UnsupportedOperationException("Not a seriable object"); } - + @Override public TimelineEvent toTimelineEvent() { TimelineEvent tEvent = new TimelineEvent(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java index 5ddd8392ec..9c0f09b017 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java @@ -228,7 +228,7 @@ public int[] getVMemKbytes() { public int[] getPhysMemKbytes() { return physMemKbytes; } - + @Override public TimelineEvent toTimelineEvent() { TimelineEvent tEvent = new TimelineEvent(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java index 30d2b0904a..a931ca2403 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java @@ -141,7 +141,7 @@ public EventType getEventType() { ? EventType.MAP_ATTEMPT_FINISHED : EventType.REDUCE_ATTEMPT_FINISHED; } - + @Override public TimelineEvent toTimelineEvent() { TimelineEvent tEvent = new TimelineEvent(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java index 99213fd77b..d09d5ca6e9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java @@ -136,7 +136,7 @@ public String getAvataar() { } return null; } - + @Override public TimelineEvent toTimelineEvent() { TimelineEvent tEvent = new TimelineEvent(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java index 8826c96847..1732d9130a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java @@ -251,7 +251,7 @@ public int[] getVMemKbytes() { public int[] getPhysMemKbytes() { return physMemKbytes; } - + @Override public TimelineEvent toTimelineEvent() { TimelineEvent tEvent = new TimelineEvent(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java index 9a29b81843..d14350df54 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java @@ -142,7 +142,7 @@ public TaskAttemptID getFailedAttemptID() { public EventType getEventType() { return EventType.TASK_FAILED; } - + @Override public TimelineEvent toTimelineEvent() { TimelineEvent tEvent = new TimelineEvent(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java index fd003ccba3..0bc4383263 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java @@ -130,7 +130,7 @@ public TimelineEvent toTimelineEvent() { tEvent.addInfo("FINISH_TIME", getFinishTime()); tEvent.addInfo("STATUS", TaskStatus.State.SUCCEEDED.toString()); tEvent.addInfo("SUCCESSFUL_TASK_ATTEMPT_ID", - getSuccessfulTaskAttemptId() == null ? "" : + getSuccessfulTaskAttemptId() == null ? "" : getSuccessfulTaskAttemptId().toString()); return tEvent; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskStartedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskStartedEvent.java index 6792ce7654..9d2fc0eee5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskStartedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskStartedEvent.java @@ -76,7 +76,7 @@ public TaskType getTaskType() { public EventType getEventType() { return EventType.TASK_STARTED; } - + @Override public TimelineEvent toTimelineEvent() { TimelineEvent tEvent = new TimelineEvent(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskUpdatedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskUpdatedEvent.java index 0722207fcc..010129d73c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskUpdatedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskUpdatedEvent.java @@ -62,7 +62,7 @@ public TaskID getTaskId() { public EventType getEventType() { return EventType.TASK_UPDATED; } - + @Override public TimelineEvent toTimelineEvent() { TimelineEvent tEvent = new TimelineEvent(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java index 27ce48ce21..6b9f27e2aa 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java @@ -65,7 +65,7 @@ public class TestMRTimelineEventHandling { private static final String TIMELINE_AUX_SERVICE_NAME = "timeline_collector"; private static final Log LOG = LogFactory.getLog(TestMRTimelineEventHandling.class); - + @Test public void testTimelineServiceStartInMiniCluster() throws Exception { Configuration conf = new YarnConfiguration(); @@ -168,7 +168,7 @@ public void testMRTimelineEventHandling() throws Exception { } } } - + @Test public void testMRNewTimelineServiceEventHandling() throws Exception { LOG.info("testMRNewTimelineServiceEventHandling start."); @@ -184,7 +184,7 @@ public void testMRNewTimelineServiceEventHandling() throws Exception { conf.set(YarnConfiguration.NM_AUX_SERVICES, TIMELINE_AUX_SERVICE_NAME); conf.set(YarnConfiguration.NM_AUX_SERVICES + "." + TIMELINE_AUX_SERVICE_NAME + ".class", PerNodeTimelineCollectorsAuxService.class.getName()); - + conf.setBoolean(YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED, true); MiniMRYarnCluster cluster = null; @@ -215,9 +215,9 @@ public void testMRNewTimelineServiceEventHandling() throws Exception { YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(new Configuration(cluster.getConfig())); yarnClient.start(); - EnumSet appStates = + EnumSet appStates = EnumSet.allOf(YarnApplicationState.class); - + ApplicationId firstAppId = null; List apps = yarnClient.getApplications(appStates); Assert.assertEquals(apps.size(), 1); @@ -230,7 +230,7 @@ public void testMRNewTimelineServiceEventHandling() throws Exception { job = UtilsForTests.runJobFail(new JobConf(conf), inDir, outDir); Assert.assertEquals(JobStatus.FAILED, job.getJobStatus().getState().getValue()); - + apps = yarnClient.getApplications(appStates); Assert.assertEquals(apps.size(), 2); @@ -250,10 +250,10 @@ public void testMRNewTimelineServiceEventHandling() throws Exception { if(testRootFolder.isDirectory()) { FileUtils.deleteDirectory(testRootFolder); } - + } } - + private void checkNewTimelineEvent(ApplicationId appId, ApplicationReport appReport) throws IOException { String tmpRoot = @@ -261,7 +261,7 @@ private void checkNewTimelineEvent(ApplicationId appId, + "/entities/"; File tmpRootFolder = new File(tmpRoot); - + Assert.assertTrue(tmpRootFolder.isDirectory()); String basePath = tmpRoot + YarnConfiguration.DEFAULT_RM_CLUSTER_ID + "/" + UserGroupInformation.getCurrentUser().getShortUserName() + @@ -319,7 +319,7 @@ private void checkNewTimelineEvent(ApplicationId appId, Assert.assertTrue("Task output directory: " + outputDirTask + " does not exist.", taskFolder.isDirectory()); - + String taskEventFileName = appId.toString().replaceAll("application", "task") + "_m_000000" + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION; @@ -330,15 +330,15 @@ private void checkNewTimelineEvent(ApplicationId appId, taskEventFile.exists()); verifyEntity(taskEventFile, EventType.TASK_FINISHED.name(), true, false, null); - + // check for task attempt event file String outputDirTaskAttempt = basePath + "/MAPREDUCE_TASK_ATTEMPT/"; File taskAttemptFolder = new File(outputDirTaskAttempt); - Assert.assertTrue("TaskAttempt output directory: " + outputDirTaskAttempt + + Assert.assertTrue("TaskAttempt output directory: " + outputDirTaskAttempt + " does not exist.", taskAttemptFolder.isDirectory()); - + String taskAttemptEventFileName = appId.toString().replaceAll( - "application", "attempt") + "_m_000000_0" + + "application", "attempt") + "_m_000000_0" + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION; String taskAttemptEventFilePath = outputDirTaskAttempt + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java index 4c42bd3f09..76198b8284 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java @@ -93,7 +93,7 @@ public MapredTestDriver(ProgramDriver pgd) { pgd.addClass("timelineperformance", TimelineServicePerformance.class, "A job that launches mappers to test timline service " + "performance."); - pgd.addClass("nnbench", NNBench.class, + pgd.addClass("nnbench", NNBench.class, "A benchmark that stresses the namenode w/ MR."); pgd.addClass("nnbenchWithoutMR", NNBenchWithoutMR.class, "A benchmark that stresses the namenode w/o MR."); diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml index 08c6ba28c7..7c19c5ee93 100644 --- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -552,9 +552,4 @@ - - - - - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 2e34ef8066..907d09ecb8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -631,7 +631,7 @@ public void run() throws YarnException, IOException, InterruptedException { DSEvent.DS_APP_ATTEMPT_START); } else { publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(), - DSEvent.DS_APP_ATTEMPT_END, domainId, appSubmitterUgi); + DSEvent.DS_APP_ATTEMPT_START, domainId, appSubmitterUgi); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java index 74da919ea1..a01d21b611 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java @@ -149,7 +149,7 @@ private void setupInternal(int numNodeManager, float timelineVersion) // Enable ContainersMonitorImpl conf.set(YarnConfiguration.NM_CONTAINER_MON_RESOURCE_CALCULATOR, LinuxResourceCalculatorPlugin.class.getName()); - conf.set(YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE, + conf.set(YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE, ProcfsBasedProcessTree.class.getName()); conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, true); conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index ef8838e1a1..478efc4a31 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -495,8 +495,8 @@ protected void putObjects( } if (resp == null || resp.getClientResponseStatus() != ClientResponse.Status.OK) { - String msg = "Response from the timeline server is " + - ((resp == null) ? "null": + String msg = "Response from the timeline server is " + + ((resp == null) ? "null": "not successful," + " HTTP error code: " + resp.getStatus() + ", Server response:\n" + resp.getEntity(String.class)); LOG.error(msg); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 944c7d6b65..ed220c0d8a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -784,7 +784,7 @@ The setting that controls whether yarn system metrics is - published to the Timeline server (version one) or not, by RM. + published to the Timeline server (version one) or not, by RM. This configuration is now deprecated in favor of yarn.system-metrics-publisher.enabled. yarn.resourcemanager.system-metrics-publisher.enabled diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimelineServiceHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimelineServiceHelper.java index f852df0a19..34b9497f80 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimelineServiceHelper.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimelineServiceHelper.java @@ -27,49 +27,49 @@ import org.junit.Test; public class TestTimelineServiceHelper { - + @Test public void testMapCastToHashMap() { // Test null map be casted to null Map nullMap = null; Assert.assertNull(TimelineServiceHelper.mapCastToHashMap(nullMap)); - + // Test empty hashmap be casted to a empty hashmap Map emptyHashMap = new HashMap(); Assert.assertEquals(TimelineServiceHelper.mapCastToHashMap(emptyHashMap).size(), 0); - + // Test empty non-hashmap be casted to a empty hashmap Map emptyTreeMap = new TreeMap(); Assert.assertEquals(TimelineServiceHelper.mapCastToHashMap(emptyTreeMap).size(), 0); - + // Test non-empty hashmap be casted to hashmap correctly Map firstHashMap = new HashMap(); String key = "KEY"; String value = "VALUE"; firstHashMap.put(key, value); Assert.assertEquals(TimelineServiceHelper.mapCastToHashMap(firstHashMap), firstHashMap); - + // Test non-empty non-hashmap is casted correctly. Map firstTreeMap = new TreeMap(); firstTreeMap.put(key, value); - HashMap alternateHashMap = + HashMap alternateHashMap = TimelineServiceHelper.mapCastToHashMap(firstTreeMap); Assert.assertEquals(firstTreeMap.size(), alternateHashMap.size()); Assert.assertEquals(alternateHashMap.get(key), value); - + // Test complicated hashmap be casted correctly Map> complicatedHashMap = new HashMap>(); Set hashSet = new HashSet(); hashSet.add(value); complicatedHashMap.put(key, hashSet); - Assert.assertEquals(TimelineServiceHelper.mapCastToHashMap(complicatedHashMap), + Assert.assertEquals(TimelineServiceHelper.mapCastToHashMap(complicatedHashMap), complicatedHashMap); - + // Test complicated non-hashmap get casted correctly Map> complicatedTreeMap = new TreeMap>(); complicatedTreeMap.put(key, hashSet); - Assert.assertEquals(TimelineServiceHelper.mapCastToHashMap(complicatedTreeMap).get(key), + Assert.assertEquals(TimelineServiceHelper.mapCastToHashMap(complicatedTreeMap).get(key), hashSet); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java index 066abfc744..912a7a3c45 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java @@ -59,7 +59,7 @@ public class NodeHeartbeatRequestPBImpl extends NodeHeartbeatRequest { private List logAggregationReportsForApps = null; Map registeredCollectors = null; - + public NodeHeartbeatRequestPBImpl() { builder = NodeHeartbeatRequestProto.newBuilder(); } @@ -157,7 +157,7 @@ private LogAggregationReportProto convertToProtoFormat( LogAggregationReport value) { return ((LogAggregationReportPBImpl) value).getProto(); } - + private void addRegisteredCollectorsToProto() { maybeInitBuilder(); builder.clearRegisteredCollectors(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java index e53c0b0430..131eaa3df0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java @@ -102,7 +102,7 @@ interface QueuingContext { NMStateStoreService getNMStateStore(); boolean getDecommissioned(); - + Configuration getConf(); void setDecommissioned(boolean isDecommissioned); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java index d48a00c129..0c84be0e59 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java @@ -471,9 +471,9 @@ public void run() { public static class NMContext implements Context { private NodeId nodeId = null; - + private Configuration conf = null; - + protected final ConcurrentMap applications = new ConcurrentHashMap(); @@ -551,7 +551,7 @@ public int getHttpPort() { public ConcurrentMap getApplications() { return this.applications; } - + @Override public Configuration getConf() { return this.conf; @@ -684,7 +684,12 @@ public Map getRegisteredCollectors() { public void addRegisteredCollectors( Map newRegisteredCollectors) { - this.registeredCollectors.putAll(newRegisteredCollectors); + if (registeredCollectors != null) { + this.registeredCollectors.putAll(newRegisteredCollectors); + } else { + LOG.warn("collectors are added when the registered collectors are " + + "initialized"); + } } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java index 22779bb4b3..5a5d30cce1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java @@ -184,7 +184,7 @@ public String getUser() { public ApplicationId getAppId() { return appId; } - + @Override public ApplicationState getApplicationState() { this.readLock.lock(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java index 6df7211248..4167ece602 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java @@ -243,7 +243,6 @@ protected void serviceStop() throws Exception { ; } } - super.serviceStop(); } @@ -422,7 +421,6 @@ public void run() { .entrySet()) { ContainerId containerId = entry.getKey(); ProcessTreeInfo ptInfo = entry.getValue(); - try { String pId = ptInfo.getPID(); @@ -478,7 +476,7 @@ public void run() { float cpuUsageTotalCoresPercentage = cpuUsagePercentPerCore / resourceCalculatorPlugin.getNumProcessors(); - + // Multiply by 1000 to avoid losing data when converting to int int milliVcoresUsed = (int) (cpuUsageTotalCoresPercentage * 1000 * maxVCoresAllottedForContainers /nodeCpuPercentageForYARN); @@ -514,7 +512,7 @@ public void run() { containerMetricsUnregisterDelayMs).recordCpuUsage ((int)cpuUsagePercentPerCore, milliVcoresUsed); } - + boolean isMemoryOverLimit = false; String msg = ""; int containerExitStatus = ContainerExitStatus.INVALID; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java index 9923f409b7..41037f7961 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java @@ -111,7 +111,6 @@ public boolean isPmemCheckEnabled() { return true; } }; - conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath()); conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath()); NodeHealthCheckerService healthChecker = createNodeHealthCheckerService(conf); @@ -175,7 +174,6 @@ public boolean isPmemCheckEnabled() { return true; } }; - conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath()); conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath()); NodeHealthCheckerService healthChecker = createNodeHealthCheckerService(conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml index 7b64ba008a..834dc3d523 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml @@ -177,7 +177,7 @@ ${project.version} - org.apache.hadoop + org.apache.hadoop hadoop-yarn-server-timelineservice diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java index 12daa959cf..caa0ff1372 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java @@ -30,10 +30,8 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; -import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; -import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; -import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMDelegatedNodeLabelsUpdater; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager; import org.apache.hadoop.yarn.server.resourcemanager.recovery.NullRMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; @@ -95,8 +93,6 @@ public class RMActiveServiceContext { private NodesListManager nodesListManager; private ResourceTrackerService resourceTrackerService; private ApplicationMasterService applicationMasterService; - private RMApplicationHistoryWriter rmApplicationHistoryWriter; - private SystemMetricsPublisher systemMetricsPublisher; private RMTimelineCollectorManager timelineCollectorManager; private RMNodeLabelsManager nodeLabelManager; @@ -373,12 +369,6 @@ public boolean isWorkPreservingRecoveryEnabled() { return this.isWorkPreservingRecoveryEnabled; } - @Private - @Unstable - public RMApplicationHistoryWriter getRMApplicationHistoryWriter() { - return rmApplicationHistoryWriter; - } - @Private @Unstable public RMTimelineCollectorManager getRMTimelineCollectorManager() { @@ -392,26 +382,6 @@ public void setRMTimelineCollectorManager( this.timelineCollectorManager = collectorManager; } - @Private - @Unstable - public void setSystemMetricsPublisher( - SystemMetricsPublisher metricsPublisher) { - this.systemMetricsPublisher = metricsPublisher; - } - - @Private - @Unstable - public SystemMetricsPublisher getSystemMetricsPublisher() { - return systemMetricsPublisher; - } - - @Private - @Unstable - public void setRMApplicationHistoryWriter( - RMApplicationHistoryWriter rmApplicationHistoryWriter) { - this.rmApplicationHistoryWriter = rmApplicationHistoryWriter; - } - @Private @Unstable public long getEpoch() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java index 3b8a5b4675..2ba445cb6a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java @@ -113,10 +113,10 @@ void setRMApplicationHistoryWriter( void setSystemMetricsPublisher(SystemMetricsPublisher systemMetricsPublisher); SystemMetricsPublisher getSystemMetricsPublisher(); - + void setRMTimelineCollectorManager( RMTimelineCollectorManager timelineCollectorManager); - + RMTimelineCollectorManager getRMTimelineCollectorManager(); ConfigurationProvider getConfigurationProvider(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java index 12a5577b41..e1e3298fda 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java @@ -380,7 +380,7 @@ public void setRMTimelineCollectorManager( public RMTimelineCollectorManager getRMTimelineCollectorManager() { return activeServiceContext.getRMTimelineCollectorManager(); } - + @Override public void setSystemMetricsPublisher( SystemMetricsPublisher metricsPublisher) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 9fa10d2f72..0c1df3392c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -22,11 +22,11 @@ import java.io.InputStream; import java.io.PrintStream; import java.net.InetSocketAddress; +import java.nio.charset.Charset; import java.security.PrivilegedExceptionAction; +import java.security.SecureRandom; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -123,15 +123,6 @@ import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.zookeeper.server.auth.DigestAuthenticationProvider; -import java.io.IOException; -import java.io.InputStream; -import java.io.PrintStream; -import java.net.InetSocketAddress; -import java.nio.charset.Charset; -import java.security.PrivilegedExceptionAction; -import java.security.SecureRandom; -import java.util.ArrayList; -import java.util.List; import com.google.common.annotations.VisibleForTesting; /** @@ -312,6 +303,15 @@ protected void serviceInit(Configuration conf) throws Exception { addService(rmApplicationHistoryWriter); rmContext.setRMApplicationHistoryWriter(rmApplicationHistoryWriter); + // initialize the RM timeline collector first so that the system metrics + // publisher can bind to it + if (YarnConfiguration.timelineServiceV2Enabled(this.conf)) { + RMTimelineCollectorManager timelineCollectorManager = + createRMTimelineCollectorManager(); + addService(timelineCollectorManager); + rmContext.setRMTimelineCollectorManager(timelineCollectorManager); + } + SystemMetricsPublisher systemMetricsPublisher = createSystemMetricsPublisher(); addIfService(systemMetricsPublisher); @@ -602,18 +602,6 @@ protected void serviceInit(Configuration configuration) throws Exception { rmContext.setDelegationTokenRenewer(delegationTokenRenewer); } - RMApplicationHistoryWriter rmApplicationHistoryWriter = - createRMApplicationHistoryWriter(); - addService(rmApplicationHistoryWriter); - rmContext.setRMApplicationHistoryWriter(rmApplicationHistoryWriter); - - if (YarnConfiguration.timelineServiceV2Enabled(configuration)) { - RMTimelineCollectorManager timelineCollectorManager = - createRMTimelineCollectorManager(); - addService(timelineCollectorManager); - rmContext.setRMTimelineCollectorManager(timelineCollectorManager); - } - // Register event handler for NodesListManager nodesListManager = new NodesListManager(rmContext); rmDispatcher.register(NodesListManagerEventType.class, nodesListManager); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java index 8a795cde59..98cbd92180 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java @@ -177,19 +177,25 @@ ApplicationReport createAndGetApplicationReport(String clientUserName, String getTrackingUrl(); /** - * The collector address for the application. - * @return the address for the application's collector. + * The collector address for the application. It should be used only if the + * timeline service v.2 is enabled. + * + * @return the address for the application's collector, or null if the + * timeline service v.2 is not enabled. */ String getCollectorAddr(); /** - * Set collector address for the application + * Set collector address for the application. It should be used only if the + * timeline service v.2 is enabled. + * * @param collectorAddr the address of collector */ void setCollectorAddr(String collectorAddr); /** - * Remove collector address when application is finished or killed. + * Remove collector address when application is finished or killed. It should + * be used only if the timeline service v.2 is enabled. */ void removeCollectorAddr(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 6e448f700b..45ff79cdfe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -526,6 +526,10 @@ public RMAppImpl(ApplicationId applicationId, RMContext rmContext, } } + /** + * Starts the application level timeline collector for this app. This should + * be used only if the timeline service v.2 is enabled. + */ public void startTimelineCollector() { AppLevelTimelineCollector collector = new AppLevelTimelineCollector(applicationId); @@ -533,6 +537,10 @@ public void startTimelineCollector() { applicationId, collector); } + /** + * Stops the application level timeline collector for this app. This should be + * used only if the timeline service v.2 is enabled. + */ public void stopTimelineCollector() { rmContext.getRMTimelineCollectorManager().remove(applicationId); }