YARN-6137. Yarn client implicitly invoke ATS client which accesses HDFS. Contributed by Li Lu

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java

@@ -136,7 +136,7 @@ public class YarnClientImpl extends YarnClient {
   private long asyncApiPollTimeoutMillis;
   protected AHSClient historyClient;
   private boolean historyServiceEnabled;
-  protected TimelineClient timelineClient;
+  protected volatile TimelineClient timelineClient;
   @VisibleForTesting
   Text timelineService;
   @VisibleForTesting
@@ -169,24 +169,9 @@ protected void serviceInit(Configuration conf) throws Exception {
 
     if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
         YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
-      try {
-        timelineServiceEnabled = true;
-        timelineClient = createTimelineClient();
-        timelineClient.init(conf);
-        timelineDTRenewer = getTimelineDelegationTokenRenewer(conf);
-        timelineService = TimelineUtils.buildTimelineTokenService(conf);
-      } catch (NoClassDefFoundError error) {
-        // When attempt to initiate the timeline client with
-        // different set of dependencies, it may fail with
-        // NoClassDefFoundError. When some of them are not compatible
-        // with timeline server. This is not necessarily a fatal error
-        // to the client.
-        LOG.warn("Timeline client could not be initialized "
-            + "because dependency missing or incompatible,"
-            + " disabling timeline client.",
-            error);
-        timelineServiceEnabled = false;
-      }
+      timelineServiceEnabled = true;
+      timelineDTRenewer = getTimelineDelegationTokenRenewer(conf);
+      timelineService = TimelineUtils.buildTimelineTokenService(conf);
     }
 
     // The AHSClientService is enabled by default when we start the
@@ -219,9 +204,6 @@ protected void serviceStart() throws Exception {
       if (historyServiceEnabled) {
         historyClient.start();
       }
-      if (timelineServiceEnabled) {
-        timelineClient.start();
-      }
     } catch (IOException e) {
       throw new YarnRuntimeException(e);
     }
@@ -236,7 +218,7 @@ protected void serviceStop() throws Exception {
     if (historyServiceEnabled) {
       historyClient.stop();
     }
-    if (timelineServiceEnabled) {
+    if (timelineClient != null) {
       timelineClient.stop();
     }
     super.serviceStop();
@@ -375,16 +357,26 @@ private void addTimelineDelegationToken(
   @VisibleForTesting
   org.apache.hadoop.security.token.Token<TimelineDelegationTokenIdentifier>
       getTimelineDelegationToken() throws IOException, YarnException {
-        try {
-          return timelineClient.getDelegationToken(timelineDTRenewer);
-        } catch (Exception e ) {
-          if (timelineServiceBestEffort) {
-            LOG.warn("Failed to get delegation token from the timeline server: "
-                + e.getMessage());
-            return null;
+    try {
+      // Only reachable when both security and timeline service are enabled.
+      if (timelineClient == null) {
+        synchronized (this) {
+          if (timelineClient == null) {
+            timelineClient = createTimelineClient();
+            timelineClient.init(getConfig());
+            timelineClient.start();
           }
-          throw e;
         }
+      }
+      return timelineClient.getDelegationToken(timelineDTRenewer);
+    } catch (Exception e) {
+      if (timelineServiceBestEffort) {
+        LOG.warn("Failed to get delegation token from the timeline server: "
+            + e.getMessage());
+        return null;
+      }
+      throw e;
+    }
   }
 
   private static String getTimelineDelegationTokenRenewer(Configuration conf)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java

@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.yarn.client.api.impl;
 
-import static org.junit.Assert.assertFalse;
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
@@ -157,25 +156,36 @@ public void testClientStop() {
   }
 
   @Test
-  public void testTimelineClientInitFailure() throws Exception{
+  public void testStartWithTimelineV15Failure() throws Exception{
     Configuration conf = new Configuration();
     conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+    conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 1.5f);
+    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT,
+        true);
     YarnClient client = YarnClient.createYarnClient();
     if(client instanceof YarnClientImpl) {
       YarnClientImpl impl = (YarnClientImpl) client;
       YarnClientImpl spyClient = spy(impl);
       when(spyClient.createTimelineClient()).thenThrow(
-          new NoClassDefFoundError(
-              "Mock a failure when init timeline instance"));
+          new IOException("ATS v1.5 client initialization failed. "));
       spyClient.init(conf);
       spyClient.start();
-      assertFalse("Timeline client should be disabled when"
-          + "it is failed to init",
-          spyClient.timelineServiceEnabled);
+      spyClient.getTimelineDelegationToken();
       spyClient.stop();
     }
   }
 
+  @Test
+  public void testStartWithTimelineV15() throws Exception {
+    Configuration conf = new Configuration();
+    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+    conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 1.5f);
+    YarnClientImpl client = (YarnClientImpl) YarnClient.createYarnClient();
+    client.init(conf);
+    client.start();
+    client.stop();
+  }
+
   @SuppressWarnings("deprecation")
   @Test (timeout = 30000)
   public void testSubmitApplication() throws Exception {