YARN-2375. Allow enabling/disabling timeline server per framework. (Mit Desai via jeagles)

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java

@@ -62,6 +62,7 @@
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
 import org.apache.hadoop.yarn.client.api.TimelineClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
@@ -243,9 +244,15 @@ protected void serviceInit(Configuration conf) throws Exception {
 
     if (conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA,
         MRJobConfig.DEFAULT_MAPREDUCE_JOB_EMIT_TIMELINE_DATA)) {
+      if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
+            YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
         timelineClient = TimelineClient.createTimelineClient();
         timelineClient.init(conf);
+        LOG.info("Timeline service is enabled");
         LOG.info("Emitting job history data to the timeline server is enabled");
+      } else {
+        LOG.info("Timeline service is not enabled");
+      }
     } else {
       LOG.info("Emitting job history data to the timeline server is not enabled");
     }

hadoop-yarn-project/CHANGES.txt

@@ -81,6 +81,9 @@ Release 2.7.0 - UNRELEASED
     YARN-2802. ClusterMetrics to include AM launch and register delays.
     (Zhihai Xu via kasha)
 
+    YARN-2375. Allow enabling/disabling timeline server per framework.
+    (Mit Desai via jeagles)
+
   OPTIMIZATIONS
 
   BUG FIXES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java

@@ -497,10 +497,16 @@ public boolean init(String[] args) throws ParseException, IOException {
     requestPriority = Integer.parseInt(cliParser
         .getOptionValue("priority", "0"));
 
+    if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
+      YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
       // Creating the Timeline Client
       timelineClient = TimelineClient.createTimelineClient();
       timelineClient.init(conf);
       timelineClient.start();
+    } else {
+      timelineClient = null;
+      LOG.warn("Timeline service is not enabled");
+    }
 
     return true;
   }
@@ -549,8 +555,10 @@ public void run() throws YarnException, IOException {
         UserGroupInformation.createRemoteUser(appSubmitterUserName);
     appSubmitterUgi.addCredentials(credentials);
     
+    if(timelineClient != null) {
       publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(),
           DSEvent.DS_APP_ATTEMPT_START, domainId, appSubmitterUgi);
+    }
 
     AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler();
     amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener);
@@ -617,9 +625,11 @@ public void run() throws YarnException, IOException {
     }
     numRequestedContainers.set(numTotalContainers);
 
+    if(timelineClient != null) {
       publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(),
           DSEvent.DS_APP_ATTEMPT_END, domainId, appSubmitterUgi);
     }
+  }
 
   @VisibleForTesting
   NMCallbackHandler createNMCallbackHandler() {
@@ -681,6 +691,11 @@ protected boolean finish() {
     
     amRMClient.stop();
 
+    // Stop Timeline Client
+    if(timelineClient != null) {
+      timelineClient.stop();
+    }
+
     return success;
   }
   
@@ -724,9 +739,11 @@ public void onContainersCompleted(List<ContainerStatus> completedContainers) {
           LOG.info("Container completed successfully." + ", containerId="
               + containerStatus.getContainerId());
         }
+        if(timelineClient != null) {
           publishContainerEndEvent(
               timelineClient, containerStatus, domainId, appSubmitterUgi);
         }
+      }
       
       // ask for more containers if any failed
       int askCount = numTotalContainers - numRequestedContainers.get();
@@ -840,10 +857,12 @@ public void onContainerStarted(ContainerId containerId,
       if (container != null) {
         applicationMaster.nmClientAsync.getContainerStatusAsync(containerId, container.getNodeId());
       }
+      if(applicationMaster.timelineClient != null) {
         ApplicationMaster.publishContainerStartEvent(
             applicationMaster.timelineClient, container,
             applicationMaster.domainId, applicationMaster.appSubmitterUgi);
       }
+    }
 
     @Override
     public void onStartContainerError(ContainerId containerId, Throwable t) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java

@@ -105,7 +105,6 @@ public class TimelineClientImpl extends TimelineClient {
   private DelegationTokenAuthenticator authenticator;
   private DelegationTokenAuthenticatedURL.Token token;
   private URI resURI;
-  private boolean isEnabled;
 
   @Private
   @VisibleForTesting
@@ -247,12 +246,6 @@ public TimelineClientImpl() {
   }
 
   protected void serviceInit(Configuration conf) throws Exception {
-    isEnabled = conf.getBoolean(
-        YarnConfiguration.TIMELINE_SERVICE_ENABLED,
-        YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED);
-    if (!isEnabled) {
-      LOG.info("Timeline service is not enabled");
-    } else {
     ClientConfig cc = new DefaultClientConfig();
     cc.getClasses().add(YarnJacksonJaxbJsonProvider.class);
     connConfigurator = newConnConfigurator(conf);
@@ -283,19 +276,12 @@ protected void serviceInit(Configuration conf) throws Exception {
           RESOURCE_URI_STR));
     }
     LOG.info("Timeline service address: " + resURI);
-    }
     super.serviceInit(conf);
   }
 
   @Override
   public TimelinePutResponse putEntities(
       TimelineEntity... entities) throws IOException, YarnException {
-    if (!isEnabled) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Nothing will be put because timeline service is not enabled");
-      }
-      return new TimelinePutResponse();
-    }
     TimelineEntities entitiesContainer = new TimelineEntities();
     entitiesContainer.addEntities(Arrays.asList(entities));
     ClientResponse resp = doPosting(entitiesContainer, null);
@@ -306,12 +292,6 @@ public TimelinePutResponse putEntities(
   @Override
   public void putDomain(TimelineDomain domain) throws IOException,
       YarnException {
-    if (!isEnabled) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Nothing will be put because timeline service is not enabled");
-      }
-      return;
-    }
     doPosting(domain, "domain");
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java

@@ -119,41 +119,6 @@ public void testPostEntitiesConnectionRefused() throws Exception {
     }
   }
 
-  @Test
-  public void testPostEntitiesTimelineServiceNotEnabled() throws Exception {
-    YarnConfiguration conf = new YarnConfiguration();
-    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
-    TimelineClientImpl client = createTimelineClient(conf);
-    mockEntityClientResponse(
-        client, ClientResponse.Status.INTERNAL_SERVER_ERROR, false, false);
-    try {
-      TimelinePutResponse response = client.putEntities(generateEntity());
-      Assert.assertEquals(0, response.getErrors().size());
-    } catch (YarnException e) {
-      Assert.fail(
-          "putEntities should already return before throwing the exception");
-    }
-  }
-
-  @Test
-  public void testPostEntitiesTimelineServiceDefaultNotEnabled()
-      throws Exception {
-    YarnConfiguration conf = new YarnConfiguration();
-    // Unset the timeline service's enabled properties.
-    // Make sure default value is pickup up
-    conf.unset(YarnConfiguration.TIMELINE_SERVICE_ENABLED);
-    TimelineClientImpl client = createTimelineClient(conf);
-    mockEntityClientResponse(client, ClientResponse.Status.INTERNAL_SERVER_ERROR,
-        false, false);
-    try {
-      TimelinePutResponse response = client.putEntities(generateEntity());
-      Assert.assertEquals(0, response.getErrors().size());
-    } catch (YarnException e) {
-      Assert
-          .fail("putEntities should already return before throwing the exception");
-    }
-  }
-
   @Test
   public void testPutDomain() throws Exception {
     mockDomainClientResponse(client, ClientResponse.Status.OK, false);