YARN-1936. Added security support for the Timeline Client. Contributed by Zhijie Shen.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1597153 13f79535-47bb-0310-9956-ffa450edef68

hadoop-yarn-project/CHANGES.txt

@@ -93,6 +93,9 @@ Release 2.5.0 - UNRELEASED
     YARN-2049. Added delegation-token support for the Timeline Server. (Zhijie
     Shen via vinodkv)
 
+    YARN-1936. Added security support for the Timeline Client. (Zhijie Shen via
+    vinodkv)
+
   OPTIMIZATIONS
 
   BUG FIXES 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java

@@ -23,11 +23,13 @@
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.service.AbstractService;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
 import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
 import org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
 
 /**
  * A client library that can be used to post some information in terms of a
@@ -65,4 +67,22 @@ protected TimelineClient(String name) {
   public abstract TimelinePutResponse putEntities(
       TimelineEntity... entities) throws IOException, YarnException;
 
+  /**
+   * <p>
+   * Get a delegation token so as to be able to talk to the timeline server in a
+   * secure way.
+   * </p>
+   * 
+   * @param renewer
+   *          Address of the renewer who can renew these tokens when needed by
+   *          securely talking to the timeline server
+   * @return a delegation token ({@link Token}) that can be used to talk to the
+   *         timeline server
+   * @throws IOException
+   * @throws YarnException
+   */
+  @Public
+  public abstract Token<TimelineDelegationTokenIdentifier> getDelegationToken(
+      String renewer) throws IOException, YarnException;
+
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java

@@ -18,24 +18,43 @@
 
 package org.apache.hadoop.yarn.client.api.impl;
 
+import java.io.File;
 import java.io.IOException;
+import java.net.HttpURLConnection;
 import java.net.URI;
+import java.net.URL;
 import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
 
 import javax.ws.rs.core.MediaType;
 
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
 import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
 import org.apache.hadoop.yarn.client.api.TimelineClient;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenSelector;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
 import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
+import org.codehaus.jackson.map.ObjectMapper;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Joiner;
@@ -44,6 +63,8 @@
 import com.sun.jersey.api.client.WebResource;
 import com.sun.jersey.api.client.config.ClientConfig;
 import com.sun.jersey.api.client.config.DefaultClientConfig;
+import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
+import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
 
 @Private
 @Unstable
@@ -52,17 +73,30 @@ public class TimelineClientImpl extends TimelineClient {
   private static final Log LOG = LogFactory.getLog(TimelineClientImpl.class);
   private static final String RESOURCE_URI_STR = "/ws/v1/timeline/";
   private static final Joiner JOINER = Joiner.on("");
+  private static Options opts;
+  static {
+    opts = new Options();
+    opts.addOption("put", true, "Put the TimelineEntities in a JSON file");
+    opts.getOption("put").setArgName("Path to the JSON file");
+    opts.addOption("help", false, "Print usage");
+  }
 
   private Client client;
   private URI resURI;
   private boolean isEnabled;
+  private TimelineAuthenticatedURLConnectionFactory urlFactory;
 
   public TimelineClientImpl() {
     super(TimelineClientImpl.class.getName());
     ClientConfig cc = new DefaultClientConfig();
     cc.getClasses().add(YarnJacksonJaxbJsonProvider.class);
+    if (UserGroupInformation.isSecurityEnabled()) {
+      urlFactory = new TimelineAuthenticatedURLConnectionFactory();
+      client = new Client(new URLConnectionClientHandler(urlFactory), cc);
+    } else {
       client = Client.create(cc);
     }
+  }
 
   protected void serviceInit(Configuration conf) throws Exception {
     isEnabled = conf.getBoolean(
@@ -83,6 +117,9 @@ protected void serviceInit(Configuration conf) throws Exception {
             YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS),
             RESOURCE_URI_STR));
       }
+      if (UserGroupInformation.isSecurityEnabled()) {
+        urlFactory.setService(TimelineUtils.buildTimelineTokenService(conf));
+      }
       LOG.info("Timeline service address: " + resURI);
     }
     super.serviceInit(conf);
@@ -124,6 +161,13 @@ public TimelinePutResponse putEntities(
     return resp.getEntity(TimelinePutResponse.class);
   }
 
+  @Override
+  public Token<TimelineDelegationTokenIdentifier> getDelegationToken(
+      String renewer) throws IOException, YarnException {
+    return TimelineAuthenticator.getDelegationToken(resURI.toURL(),
+        urlFactory.token, renewer);
+  }
+
   @Private
   @VisibleForTesting
   public ClientResponse doPostingEntities(TimelineEntities entities) {
@@ -133,4 +177,138 @@ public ClientResponse doPostingEntities(TimelineEntities entities) {
         .post(ClientResponse.class, entities);
   }
 
+  private static class TimelineAuthenticatedURLConnectionFactory
+      implements HttpURLConnectionFactory {
+
+    private AuthenticatedURL.Token token;
+    private TimelineAuthenticator authenticator;
+    private Token<TimelineDelegationTokenIdentifier> dToken;
+    private Text service;
+
+    public TimelineAuthenticatedURLConnectionFactory() {
+      token = new AuthenticatedURL.Token();
+      authenticator = new TimelineAuthenticator();
+    }
+
+    @Override
+    public HttpURLConnection getHttpURLConnection(URL url) throws IOException {
+      try {
+        if (dToken == null) {
+          //TODO: need to take care of the renew case
+          dToken = selectToken();
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Timeline delegation token: " + dToken.toString());
+          }
+        }
+        if (dToken != null) {
+          Map<String, String> params = new HashMap<String, String>();
+          TimelineAuthenticator.injectDelegationToken(params, dToken);
+          url = TimelineAuthenticator.appendParams(url, params);
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("URL with delegation token: " + url);
+          }
+        }
+        return new AuthenticatedURL(authenticator).openConnection(url, token);
+      } catch (AuthenticationException e) {
+        LOG.error("Authentication failed when openning connection [" + url
+            + "] with token [" + token + "].", e);
+        throw new IOException(e);
+      }
+    }
+
+    private Token<TimelineDelegationTokenIdentifier> selectToken() {
+      UserGroupInformation ugi;
+      try {
+        ugi = UserGroupInformation.getCurrentUser();
+      } catch (IOException e) {
+        String msg = "Error when getting the current user";
+        LOG.error(msg, e);
+        throw new YarnRuntimeException(msg, e);
+      }
+      TimelineDelegationTokenSelector tokenSelector =
+          new TimelineDelegationTokenSelector();
+      return tokenSelector.selectToken(
+          service, ugi.getCredentials().getAllTokens());
+    }
+
+    public void setService(Text service) {
+      this.service = service;
+    }
+
+  }
+
+  public static void main(String[] argv) throws Exception {
+    CommandLine cliParser = new GnuParser().parse(opts, argv);
+    if (cliParser.hasOption("put")) {
+      String path = cliParser.getOptionValue("put");
+      if (path != null && path.length() > 0) {
+        putTimelineEntitiesInJSONFile(path);
+        return;
+      }
+    }
+    printUsage();
+  }
+
+  /**
+   * Put timeline data in a JSON file via command line.
+   * 
+   * @param path
+   *          path to the {@link TimelineEntities} JSON file
+   */
+  private static void putTimelineEntitiesInJSONFile(String path) {
+    File jsonFile = new File(path);
+    if (!jsonFile.exists()) {
+      System.out.println("Error: File [" + jsonFile.getAbsolutePath()
+          + "] doesn't exist");
+      return;
+    }
+    ObjectMapper mapper = new ObjectMapper();
+    YarnJacksonJaxbJsonProvider.configObjectMapper(mapper);
+    TimelineEntities entities = null;
+    try {
+      entities = mapper.readValue(jsonFile, TimelineEntities.class);
+    } catch (Exception e) {
+      System.err.println("Error: " + e.getMessage());
+      e.printStackTrace(System.err);
+      return;
+    }
+    Configuration conf = new YarnConfiguration();
+    TimelineClient client = TimelineClient.createTimelineClient();
+    client.init(conf);
+    client.start();
+    try {
+      if (UserGroupInformation.isSecurityEnabled()
+          && conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false)) {
+        Token<TimelineDelegationTokenIdentifier> token =
+            client.getDelegationToken(
+                UserGroupInformation.getCurrentUser().getUserName());
+        UserGroupInformation.getCurrentUser().addToken(token);
+      }
+      TimelinePutResponse response = client.putEntities(
+          entities.getEntities().toArray(
+              new TimelineEntity[entities.getEntities().size()]));
+      if (response.getErrors().size() == 0) {
+        System.out.println("Timeline data is successfully put");
+      } else {
+        for (TimelinePutResponse.TimelinePutError error : response.getErrors()) {
+          System.out.println("TimelineEntity [" + error.getEntityType() + ":" +
+              error.getEntityId() + "] is not successfully put. Error code: " +
+              error.getErrorCode());
+        }
+      }
+    } catch (Exception e) {
+      System.err.println("Error: " + e.getMessage());
+      e.printStackTrace(System.err);
+    } finally {
+      client.stop();
+    }
+  }
+
+  /**
+   * Helper function to print out usage
+   */
+  private static void printUsage() {
+    new HelpFormatter().printHelp("TimelineClient", opts);
+  }
+
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java

@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.client.api.impl;
 
 import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.EnumSet;
 import java.util.List;
@@ -29,8 +30,13 @@
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.DataInputByteBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse;
@@ -64,6 +70,7 @@
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.api.records.ContainerReport;
 import org.apache.hadoop.yarn.api.records.NodeReport;
 import org.apache.hadoop.yarn.api.records.NodeState;
@@ -74,6 +81,7 @@
 import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
 import org.apache.hadoop.yarn.client.ClientRMProxy;
 import org.apache.hadoop.yarn.client.api.AHSClient;
+import org.apache.hadoop.yarn.client.api.TimelineClient;
 import org.apache.hadoop.yarn.client.api.YarnClient;
 import org.apache.hadoop.yarn.client.api.YarnClientApplication;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -82,8 +90,10 @@
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
+import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
 
 import com.google.common.annotations.VisibleForTesting;
 
@@ -97,8 +107,11 @@ public class YarnClientImpl extends YarnClient {
   protected long submitPollIntervalMillis;
   private long asyncApiPollIntervalMillis;
   private long asyncApiPollTimeoutMillis;
-  protected AHSClient historyClient;
+  private AHSClient historyClient;
   private boolean historyServiceEnabled;
+  protected TimelineClient timelineClient;
+  protected Text timelineService;
+  protected boolean timelineServiceEnabled;
 
   private static final String ROOT = "root";
 
@@ -126,10 +139,17 @@ protected void serviceInit(Configuration conf) throws Exception {
     if (conf.getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
       YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) {
       historyServiceEnabled = true;
-      historyClient = AHSClientImpl.createAHSClient();
-      historyClient.init(getConfig());
+      historyClient = AHSClient.createAHSClient();
+      historyClient.init(conf);
     }
 
+    if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
+        YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
+      timelineServiceEnabled = true;
+      timelineClient = TimelineClient.createTimelineClient();
+      timelineClient.init(conf);
+      timelineService = TimelineUtils.buildTimelineTokenService(conf);
+    }
     super.serviceInit(conf);
   }
 
@@ -141,6 +161,9 @@ protected void serviceStart() throws Exception {
       if (historyServiceEnabled) {
         historyClient.start();
       }
+      if (timelineServiceEnabled) {
+        timelineClient.start();
+      }
     } catch (IOException e) {
       throw new YarnRuntimeException(e);
     }
@@ -155,6 +178,9 @@ protected void serviceStop() throws Exception {
     if (historyServiceEnabled) {
       historyClient.stop();
     }
+    if (timelineServiceEnabled) {
+      timelineClient.stop();
+    }
     super.serviceStop();
   }
 
@@ -189,6 +215,12 @@ public YarnClientApplication createApplication()
         Records.newRecord(SubmitApplicationRequest.class);
     request.setApplicationSubmissionContext(appContext);
 
+    // Automatically add the timeline DT into the CLC
+    // Only when the security and the timeline service are both enabled
+    if (isSecurityEnabled() && timelineServiceEnabled) {
+      addTimelineDelegationToken(appContext.getAMContainerSpec());
+    }
+
     //TODO: YARN-1763:Handle RM failovers during the submitApplication call.
     rmClient.submitApplication(request);
 
@@ -238,6 +270,48 @@ public YarnClientApplication createApplication()
     return applicationId;
   }
 
+  private void addTimelineDelegationToken(
+      ContainerLaunchContext clc) throws YarnException, IOException {
+    org.apache.hadoop.security.token.Token<TimelineDelegationTokenIdentifier> timelineDelegationToken =
+        timelineClient.getDelegationToken(
+            UserGroupInformation.getCurrentUser().getUserName());
+    if (timelineDelegationToken == null) {
+      return;
+    }
+    Credentials credentials = new Credentials();
+    DataInputByteBuffer dibb = new DataInputByteBuffer();
+    ByteBuffer tokens = clc.getTokens();
+    if (tokens != null) {
+      dibb.reset(tokens);
+      credentials.readTokenStorageStream(dibb);
+      tokens.rewind();
+    }
+    // If the timeline delegation token is already in the CLC, no need to add
+    // one more
+    for (org.apache.hadoop.security.token.Token<? extends TokenIdentifier> token : credentials
+        .getAllTokens()) {
+      TokenIdentifier tokenIdentifier = token.decodeIdentifier();
+      if (tokenIdentifier instanceof TimelineDelegationTokenIdentifier) {
+        return;
+      }
+    }
+    credentials.addToken(timelineService, timelineDelegationToken);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Add timline delegation token into credentials: "
+          + timelineDelegationToken);
+    }
+    DataOutputBuffer dob = new DataOutputBuffer();
+    credentials.writeTokenStorageToStream(dob);
+    tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
+    clc.setTokens(tokens);
+  }
+
+  @Private
+  @VisibleForTesting
+  protected boolean isSecurityEnabled() {
+    return UserGroupInformation.isSecurityEnabled();
+  }
+
   @Override
   public void killApplication(ApplicationId applicationId)
       throws YarnException, IOException {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java

@@ -25,19 +25,26 @@
 import static org.mockito.Mockito.when;
 
 import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
-import org.junit.Assert;
-
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.DataInputByteBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse;
@@ -69,19 +76,23 @@
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.TimelineClient;
 import org.apache.hadoop.yarn.client.api.YarnClient;
 import org.apache.hadoop.yarn.client.api.YarnClientApplication;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.ApplicationIdNotProvidedException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
 import org.apache.hadoop.yarn.server.MiniYARNCluster;
 import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
 import org.apache.log4j.Level;
 import org.apache.log4j.LogManager;
 import org.apache.log4j.Logger;
+import org.junit.Assert;
 import org.junit.Test;
 
 public class TestYarnClient {
@@ -725,4 +736,80 @@ private void testAsyncAPIPollTimeoutHelper(Long valueForTimeout,
       IOUtils.closeQuietly(client);
     }
   }
+
+  @Test
+  public void testAutomaticTimelineDelegationTokenLoading()
+      throws Exception {
+    Configuration conf = new YarnConfiguration();
+    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+    SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf);
+    final Token<TimelineDelegationTokenIdentifier> dToken =
+        new Token<TimelineDelegationTokenIdentifier>();
+    // crate a mock client
+    YarnClientImpl client = new YarnClientImpl() {
+      @Override
+      protected void serviceInit(Configuration conf) throws Exception {
+        if (getConfig().getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
+            YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
+          timelineServiceEnabled = true;
+          timelineClient = mock(TimelineClient.class);
+          when(timelineClient.getDelegationToken(any(String.class)))
+              .thenReturn(dToken);
+          timelineClient.init(getConfig());
+          timelineService = TimelineUtils.buildTimelineTokenService(getConfig());
+        }
+        this.setConfig(conf);
+      }
+
+      @Override
+      protected void serviceStart() throws Exception {
+        rmClient = mock(ApplicationClientProtocol.class);
+      }
+
+      @Override
+      protected void serviceStop() throws Exception {
+      }
+
+      @Override
+      public ApplicationReport getApplicationReport(ApplicationId appId) {
+        ApplicationReport report = mock(ApplicationReport.class);
+        when(report.getYarnApplicationState())
+            .thenReturn(YarnApplicationState.SUBMITTED);
+        return report;
+      }
+
+      @Override
+      public boolean isSecurityEnabled() {
+        return true;
+      }
+    };
+    client.init(conf);
+    client.start();
+    ApplicationSubmissionContext context =
+        mock(ApplicationSubmissionContext.class);
+    ApplicationId applicationId = ApplicationId.newInstance(0, 1);
+    when(context.getApplicationId()).thenReturn(applicationId);
+    DataOutputBuffer dob = new DataOutputBuffer();
+    Credentials credentials = new Credentials();
+    credentials.writeTokenStorageToStream(dob);
+    ByteBuffer tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
+    ContainerLaunchContext clc = ContainerLaunchContext.newInstance(
+        null, null, null, null, tokens, null);
+    when(context.getAMContainerSpec()).thenReturn(clc);
+    client.submitApplication(context);
+    // Check whether token is added or not
+    credentials = new Credentials();
+    DataInputByteBuffer dibb = new DataInputByteBuffer();
+    tokens = clc.getTokens();
+    if (tokens != null) {
+      dibb.reset(tokens);
+      credentials.readTokenStorageStream(dibb);
+      tokens.rewind();
+    }
+    Collection<Token<? extends TokenIdentifier>> dTokens =
+        credentials.getAllTokens();
+    Assert.assertEquals(1, dTokens.size());
+    Assert.assertEquals(dToken, dTokens.iterator().next());
+    client.stop();
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java

@@ -19,9 +19,14 @@
 package org.apache.hadoop.yarn.util.timeline;
 
 import java.io.IOException;
+import java.net.InetSocketAddress;
 
 import org.apache.hadoop.classification.InterfaceAudience.Public;
 import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
 import org.codehaus.jackson.JsonGenerationException;
 import org.codehaus.jackson.map.JsonMappingException;
@@ -78,4 +83,26 @@ public static String dumpTimelineRecordtoJSON(Object o, boolean pretty)
     }
   }
 
+  public static InetSocketAddress getTimelineTokenServiceAddress(
+      Configuration conf) {
+    InetSocketAddress timelineServiceAddr = null;
+    if (YarnConfiguration.useHttps(conf)) {
+      timelineServiceAddr = conf.getSocketAddr(
+          YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS,
+          YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS,
+          YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT);
+    } else {
+      timelineServiceAddr = conf.getSocketAddr(
+          YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
+          YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS,
+          YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_PORT);
+    }
+    return timelineServiceAddr;
+  }
+
+  public static Text buildTimelineTokenService(Configuration conf) {
+    InetSocketAddress timelineServiceAddr =
+        getTimelineTokenServiceAddress(conf);
+    return SecurityUtil.buildTokenService(timelineServiceAddr);
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/security/TimelineDelegationTokenSecretManagerService.java

@@ -34,6 +34,7 @@
 import org.apache.hadoop.service.AbstractService;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
 
 /**
  * The service wrapper of {@link TimelineDelegationTokenSecretManager}
@@ -65,17 +66,7 @@ protected void serviceInit(Configuration conf) throws Exception {
         3600000);
     secretManager.startThreads();
 
-    if (YarnConfiguration.useHttps(getConfig())) {
-      serviceAddr = getConfig().getSocketAddr(
-          YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS,
-          YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS,
-          YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT);
-    } else {
-      serviceAddr = getConfig().getSocketAddr(
-          YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
-          YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS,
-          YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_PORT);
-    }
+    serviceAddr = TimelineUtils.getTimelineTokenServiceAddress(getConfig());
     super.init(conf);
   }