YARN-5243. fix several rebase and other miscellaneous issues before merge. (Sangjin Lee via Varun Saxena)

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java

@@ -822,6 +822,7 @@ private List<Container> getResources() throws Exception {
 
     handleUpdatedNodes(response);
     handleJobPriorityChange(response);
+    // handle receiving the timeline collector address for this app
     String collectorAddr = response.getCollectorAddr();
     MRAppMaster.RunningAppContext appContext =
         (MRAppMaster.RunningAppContext)this.getContext();

hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml

@@ -552,9 +552,4 @@
     <Package name="org.apache.hadoop.yarn.api.records.impl.pb" />
     <Bug pattern="NP_BOOLEAN_RETURN_NULL" />
   </Match>
-  <!-- Object cast is based on the event type -->
-  <Match>
-    <Class name="org.apache.hadoop.yarn.server.resourcemanager.metrics.AbstractTimelineServicePublisher" />
-     <Bug pattern="BC_UNCONFIRMED_CAST" />
-  </Match>
 </FindBugsFilter>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java

@@ -631,7 +631,7 @@ public void run() throws YarnException, IOException, InterruptedException {
             DSEvent.DS_APP_ATTEMPT_START);
       } else {
         publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(),
-          DSEvent.DS_APP_ATTEMPT_END, domainId, appSubmitterUgi);
+            DSEvent.DS_APP_ATTEMPT_START, domainId, appSubmitterUgi);
       }
     }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java

@@ -684,7 +684,12 @@ public Map<ApplicationId, String> getRegisteredCollectors() {
 
     public void addRegisteredCollectors(
         Map<ApplicationId, String> newRegisteredCollectors) {
-      this.registeredCollectors.putAll(newRegisteredCollectors);
+      if (registeredCollectors != null) {
+        this.registeredCollectors.putAll(newRegisteredCollectors);
+      } else {
+        LOG.warn("collectors are added when the registered collectors are " +
+            "initialized");
+      }
     }
 
     @Override

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java

@@ -243,7 +243,6 @@ protected void serviceStop() throws Exception {
         ;
       }
     }
-    
     super.serviceStop();
   }
 
@@ -422,7 +421,6 @@ public void run() {
             .entrySet()) {
           ContainerId containerId = entry.getKey();
           ProcessTreeInfo ptInfo = entry.getValue();
-          
           try {
             String pId = ptInfo.getPID();
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java

@@ -111,7 +111,6 @@ public boolean isPmemCheckEnabled() {
         return true;
       }
     };
-    
     conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
     conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
     NodeHealthCheckerService healthChecker = createNodeHealthCheckerService(conf);
@@ -175,7 +174,6 @@ public boolean isPmemCheckEnabled() {
         return true;
       }
     };
-    
     conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
     conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
     NodeHealthCheckerService healthChecker = createNodeHealthCheckerService(conf);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java

@@ -30,10 +30,8 @@
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
-import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
-import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMDelegatedNodeLabelsUpdater;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.NullRMStateStore;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
@@ -95,8 +93,6 @@ public class RMActiveServiceContext {
   private NodesListManager nodesListManager;
   private ResourceTrackerService resourceTrackerService;
   private ApplicationMasterService applicationMasterService;
-  private RMApplicationHistoryWriter rmApplicationHistoryWriter;
-  private SystemMetricsPublisher systemMetricsPublisher;
   private RMTimelineCollectorManager timelineCollectorManager;
 
   private RMNodeLabelsManager nodeLabelManager;
@@ -373,12 +369,6 @@ public boolean isWorkPreservingRecoveryEnabled() {
     return this.isWorkPreservingRecoveryEnabled;
   }
 
-  @Private
-  @Unstable
-  public RMApplicationHistoryWriter getRMApplicationHistoryWriter() {
-    return rmApplicationHistoryWriter;
-  }
-
   @Private
   @Unstable
   public RMTimelineCollectorManager getRMTimelineCollectorManager() {
@@ -392,26 +382,6 @@ public void setRMTimelineCollectorManager(
     this.timelineCollectorManager = collectorManager;
   }
 
-  @Private
-  @Unstable
-  public void setSystemMetricsPublisher(
-      SystemMetricsPublisher metricsPublisher) {
-    this.systemMetricsPublisher = metricsPublisher;
-  }
-
-  @Private
-  @Unstable
-  public SystemMetricsPublisher getSystemMetricsPublisher() {
-    return systemMetricsPublisher;
-  }
-
-  @Private
-  @Unstable
-  public void setRMApplicationHistoryWriter(
-      RMApplicationHistoryWriter rmApplicationHistoryWriter) {
-    this.rmApplicationHistoryWriter = rmApplicationHistoryWriter;
-  }
-
   @Private
   @Unstable
   public long getEpoch() {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java

@@ -22,11 +22,11 @@
 import java.io.InputStream;
 import java.io.PrintStream;
 import java.net.InetSocketAddress;
+import java.nio.charset.Charset;
 import java.security.PrivilegedExceptionAction;
+import java.security.SecureRandom;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -123,15 +123,6 @@
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 import org.apache.zookeeper.server.auth.DigestAuthenticationProvider;
 
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.PrintStream;
-import java.net.InetSocketAddress;
-import java.nio.charset.Charset;
-import java.security.PrivilegedExceptionAction;
-import java.security.SecureRandom;
-import java.util.ArrayList;
-import java.util.List;
 import com.google.common.annotations.VisibleForTesting;
 
 /**
@@ -312,6 +303,15 @@ protected void serviceInit(Configuration conf) throws Exception {
     addService(rmApplicationHistoryWriter);
     rmContext.setRMApplicationHistoryWriter(rmApplicationHistoryWriter);
 
+    // initialize the RM timeline collector first so that the system metrics
+    // publisher can bind to it
+    if (YarnConfiguration.timelineServiceV2Enabled(this.conf)) {
+      RMTimelineCollectorManager timelineCollectorManager =
+          createRMTimelineCollectorManager();
+      addService(timelineCollectorManager);
+      rmContext.setRMTimelineCollectorManager(timelineCollectorManager);
+    }
+
     SystemMetricsPublisher systemMetricsPublisher =
         createSystemMetricsPublisher();
     addIfService(systemMetricsPublisher);
@@ -602,18 +602,6 @@ protected void serviceInit(Configuration configuration) throws Exception {
         rmContext.setDelegationTokenRenewer(delegationTokenRenewer);
       }
 
-      RMApplicationHistoryWriter rmApplicationHistoryWriter =
-          createRMApplicationHistoryWriter();
-      addService(rmApplicationHistoryWriter);
-      rmContext.setRMApplicationHistoryWriter(rmApplicationHistoryWriter);
-
-      if (YarnConfiguration.timelineServiceV2Enabled(configuration)) {
-        RMTimelineCollectorManager timelineCollectorManager =
-            createRMTimelineCollectorManager();
-        addService(timelineCollectorManager);
-        rmContext.setRMTimelineCollectorManager(timelineCollectorManager);
-      }
-
       // Register event handler for NodesListManager
       nodesListManager = new NodesListManager(rmContext);
       rmDispatcher.register(NodesListManagerEventType.class, nodesListManager);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java

@@ -177,19 +177,25 @@ ApplicationReport createAndGetApplicationReport(String clientUserName,
   String getTrackingUrl();
 
   /**
-   * The collector address for the application.
-   * @return the address for the application's collector.
+   * The collector address for the application. It should be used only if the
+   * timeline service v.2 is enabled.
+   *
+   * @return the address for the application's collector, or null if the
+   * timeline service v.2 is not enabled.
    */
   String getCollectorAddr();
 
   /**
-   * Set collector address for the application
+   * Set collector address for the application. It should be used only if the
+   * timeline service v.2 is enabled.
+   *
    * @param collectorAddr the address of collector
    */
   void setCollectorAddr(String collectorAddr);
 
   /**
-   * Remove collector address when application is finished or killed.
+   * Remove collector address when application is finished or killed. It should
+   * be used only if the timeline service v.2 is enabled.
    */
   void removeCollectorAddr();
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java

@@ -526,6 +526,10 @@ public RMAppImpl(ApplicationId applicationId, RMContext rmContext,
     }
   }
 
+  /**
+   * Starts the application level timeline collector for this app. This should
+   * be used only if the timeline service v.2 is enabled.
+   */
   public void startTimelineCollector() {
     AppLevelTimelineCollector collector =
         new AppLevelTimelineCollector(applicationId);
@@ -533,6 +537,10 @@ public void startTimelineCollector() {
         applicationId, collector);
   }
 
+  /**
+   * Stops the application level timeline collector for this app. This should be
+   * used only if the timeline service v.2 is enabled.
+   */
   public void stopTimelineCollector() {
     rmContext.getRMTimelineCollectorManager().remove(applicationId);
   }