HADOOP-9567. Provide auto-renewal for keytab based logins. Contributed by Hrishikesh Gadre, Gary Helmling and Harsh J.

Signed-off-by: Wei-Chiu Chuang <weichiu@apache.org>

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java

@@ -636,6 +636,18 @@ public class CommonConfigurationKeysPublic {
   /** Default value for HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN */
   public static final int HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN_DEFAULT =
           60;
+
+  /**
+   * @see
+   * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+   * core-default.xml</a>
+   */
+  public static final String HADOOP_KERBEROS_KEYTAB_LOGIN_AUTORENEWAL_ENABLED =
+          "hadoop.kerberos.keytab.login.autorenewal.enabled";
+  /** Default value for HADOOP_KERBEROS_KEYTAB_LOGIN_AUTORENEWAL_ENABLED. */
+  public static final boolean
+          HADOOP_KERBEROS_KEYTAB_LOGIN_AUTORENEWAL_ENABLED_DEFAULT = false;
+
   /**
    * @see
    * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java

@@ -20,6 +20,8 @@
 import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_USER_GROUP_METRICS_PERCENTILES_INTERVALS;
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN;
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_KERBEROS_KEYTAB_LOGIN_AUTORENEWAL_ENABLED;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_KERBEROS_KEYTAB_LOGIN_AUTORENEWAL_ENABLED_DEFAULT;
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_TOKEN_FILES;
 import static org.apache.hadoop.security.UGIExceptionMessages.*;
 import static org.apache.hadoop.util.PlatformName.IBM_JAVA;
@@ -46,7 +48,11 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
@@ -280,6 +286,11 @@ public static void reattachMetrics() {
   private static Groups groups;
   /** Min time (in seconds) before relogin for Kerberos */
   private static long kerberosMinSecondsBeforeRelogin;
+  /** Boolean flag to enable auto-renewal for keytab based loging. */
+  private static boolean kerberosKeyTabLoginRenewalEnabled;
+  /** A reference to Kerberos login auto renewal thread. */
+  private static Optional<ExecutorService> kerberosLoginRenewalExecutor =
+          Optional.empty();
   /** The configuration to use */
 
   private static Configuration conf;
@@ -332,6 +343,11 @@ private static synchronized void initialize(Configuration conf,
                 HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN + " of " +
                 conf.get(HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN));
     }
+
+    kerberosKeyTabLoginRenewalEnabled = conf.getBoolean(
+            HADOOP_KERBEROS_KEYTAB_LOGIN_AUTORENEWAL_ENABLED,
+            HADOOP_KERBEROS_KEYTAB_LOGIN_AUTORENEWAL_ENABLED_DEFAULT);
+
     // If we haven't set up testing groups, use the configuration to find it
     if (!(groups instanceof TestingGroups)) {
       groups = Groups.getUserToGroupsMappingService(conf);
@@ -372,6 +388,8 @@ public static void reset() {
     conf = null;
     groups = null;
     kerberosMinSecondsBeforeRelogin = 0;
+    kerberosKeyTabLoginRenewalEnabled = false;
+    kerberosLoginRenewalExecutor = Optional.empty();
     setLoginUser(null);
     HadoopKerberosName.setRules(null);
   }
@@ -392,7 +410,23 @@ private static boolean isAuthenticationMethodEnabled(AuthenticationMethod method
     ensureInitialized();
     return (authenticationMethod == method);
   }
-  
+
+  @InterfaceAudience.Private
+  @InterfaceStability.Evolving
+  @VisibleForTesting
+  static boolean isKerberosKeyTabLoginRenewalEnabled() {
+    ensureInitialized();
+    return kerberosKeyTabLoginRenewalEnabled;
+  }
+
+  @InterfaceAudience.Private
+  @InterfaceStability.Evolving
+  @VisibleForTesting
+  static Optional<ExecutorService> getKerberosLoginRenewalExecutor() {
+    ensureInitialized();
+    return kerberosLoginRenewalExecutor;
+  }
+
   /**
    * Information about the logged in user.
    */
@@ -838,14 +872,16 @@ public boolean shouldRelogin() {
     return hasKerberosCredentials() && isHadoopLogin();
   }
 
+  /**
+   * Spawn a thread to do periodic renewals of kerberos credentials. NEVER
+   * directly call this method. This method should only be used for ticket cache
+   * based kerberos credentials.
+   *
+   * @param force - used by tests to forcibly spawn thread
+   */
   @InterfaceAudience.Private
   @InterfaceStability.Unstable
   @VisibleForTesting
-  /**
-   * Spawn a thread to do periodic renewals of kerberos credentials from
-   * a ticket cache.  NEVER directly call this method.
-   * @param force - used by tests to forcibly spawn thread
-   */
   void spawnAutoRenewalThreadForUserCreds(boolean force) {
     if (!force && (!shouldRelogin() || isFromKeytab())) {
       return;
@@ -858,25 +894,71 @@ void spawnAutoRenewalThreadForUserCreds(boolean force) {
     }
     String cmd = conf.get("hadoop.kerberos.kinit.command", "kinit");
     long nextRefresh = getRefreshTime(tgt);
-    Thread t =
+    executeAutoRenewalTask(getUserName(),
-        new Thread(new AutoRenewalForUserCredsRunnable(tgt, cmd, nextRefresh));
+            new TicketCacheRenewalRunnable(tgt, cmd, nextRefresh));
-    t.setDaemon(true);
-    t.setName("TGT Renewer for " + getUserName());
-    t.start();
   }
 
+  /**
+   * Spawn a thread to do periodic renewals of kerberos credentials from a
+   * keytab file.
+   */
+  private void spawnAutoRenewalThreadForKeytab() {
+    if (!shouldRelogin() || isFromTicket()) {
+      return;
+    }
+
+    // spawn thread only if we have kerb credentials
+    KerberosTicket tgt = getTGT();
+    if (tgt == null) {
+      return;
+    }
+    long nextRefresh = getRefreshTime(tgt);
+    executeAutoRenewalTask(getUserName(),
+            new KeytabRenewalRunnable(tgt, nextRefresh));
+  }
+
+  /**
+   * Spawn a thread to do periodic renewals of kerberos credentials from a
+   * keytab file. NEVER directly call this method.
+   *
+   * @param userName Name of the user for which login needs to be renewed.
+   * @param task  The reference of the login renewal task.
+   */
+  private void executeAutoRenewalTask(final String userName,
+                                      AutoRenewalForUserCredsRunnable task) {
+    kerberosLoginRenewalExecutor = Optional.of(
+            Executors.newSingleThreadExecutor(
+                  new ThreadFactory() {
+                    @Override
+                    public Thread newThread(Runnable r) {
+                      Thread t = new Thread(r);
+                      t.setDaemon(true);
+                      t.setName("TGT Renewer for " + userName);
+                      return t;
+                    }
+                  }
+            ));
+    kerberosLoginRenewalExecutor.get().submit(task);
+  }
+
+  /**
+   * An abstract class which encapsulates the functionality required to
+   * auto renew Kerbeors TGT. The concrete implementations of this class
+   * are expected to provide implementation required to perform actual
+   * TGT renewal (see {@code TicketCacheRenewalRunnable} and
+   * {@code KeytabRenewalRunnable}).
+   */
+  @InterfaceAudience.Private
+  @InterfaceStability.Unstable
   @VisibleForTesting
-  class AutoRenewalForUserCredsRunnable implements Runnable {
+  abstract class AutoRenewalForUserCredsRunnable implements Runnable {
     private KerberosTicket tgt;
     private RetryPolicy rp;
-    private String kinitCmd;
     private long nextRefresh;
     private boolean runRenewalLoop = true;
 
-    AutoRenewalForUserCredsRunnable(KerberosTicket tgt, String kinitCmd,
+    AutoRenewalForUserCredsRunnable(KerberosTicket tgt, long nextRefresh) {
-        long nextRefresh){
       this.tgt = tgt;
-      this.kinitCmd = kinitCmd;
       this.nextRefresh = nextRefresh;
       this.rp = null;
     }
@@ -885,6 +967,13 @@ public void setRunRenewalLoop(boolean runRenewalLoop) {
       this.runRenewalLoop = runRenewalLoop;
     }
 
+    /**
+     * This method is used to perform renewal of kerberos login ticket.
+     * The concrete implementations of this class should provide specific
+     * logic required to perform renewal as part of this method.
+     */
+    protected abstract void relogin() throws IOException;
+
     @Override
     public void run() {
       do {
@@ -897,11 +986,7 @@ public void run() {
           if (now < nextRefresh) {
             Thread.sleep(nextRefresh - now);
           }
-          String output = Shell.execCommand(kinitCmd, "-R");
+          relogin();
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("Renewed ticket. kinit output: {}", output);
-          }
-          reloginFromTicketCache();
           tgt = getTGT();
           if (tgt == null) {
             LOG.warn("No TGT after renewal. Aborting renew thread for " +
@@ -971,6 +1056,52 @@ public void run() {
     }
   }
 
+  /**
+   * A concrete implementation of {@code AutoRenewalForUserCredsRunnable} class
+   * which performs TGT renewal using kinit command.
+   */
+  @InterfaceAudience.Private
+  @InterfaceStability.Unstable
+  @VisibleForTesting
+  final class TicketCacheRenewalRunnable
+      extends AutoRenewalForUserCredsRunnable {
+    private String kinitCmd;
+
+    TicketCacheRenewalRunnable(KerberosTicket tgt, String kinitCmd,
+        long nextRefresh) {
+      super(tgt, nextRefresh);
+      this.kinitCmd = kinitCmd;
+    }
+
+    @Override
+    public void relogin() throws IOException {
+      String output = Shell.execCommand(kinitCmd, "-R");
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Renewed ticket. kinit output: {}", output);
+      }
+      reloginFromTicketCache();
+    }
+  }
+
+  /**
+   * A concrete implementation of {@code AutoRenewalForUserCredsRunnable} class
+   * which performs TGT renewal using specified keytab.
+   */
+  @InterfaceAudience.Private
+  @InterfaceStability.Unstable
+  @VisibleForTesting
+  final class KeytabRenewalRunnable extends AutoRenewalForUserCredsRunnable {
+
+    KeytabRenewalRunnable(KerberosTicket tgt, long nextRefresh) {
+      super(tgt, nextRefresh);
+    }
+
+    @Override
+    public void relogin() throws IOException {
+      reloginFromKeytab();
+    }
+  }
+
   /**
    * Get time for next login retry. This will allow the thread to retry with
    * exponential back-off, until tgt endtime.
@@ -1007,9 +1138,16 @@ static void loginUserFromKeytab(String user,
     if (!isSecurityEnabled())
       return;
 
-    setLoginUser(loginUserFromKeytabAndReturnUGI(user, path));
+    UserGroupInformation u = loginUserFromKeytabAndReturnUGI(user, path);
-    LOG.info("Login successful for user " + user
+    if (isKerberosKeyTabLoginRenewalEnabled()) {
-        + " using keytab file " + path);
+      u.spawnAutoRenewalThreadForKeytab();
+    }
+
+    setLoginUser(u);
+
+    LOG.info("Login successful for user {} using keytab file {}. Keytab auto" +
+            " renewal enabled : {}",
+            user, path, isKerberosKeyTabLoginRenewalEnabled());
   }
 
   /**
@@ -1027,6 +1165,12 @@ public void logoutUserFromKeytab() throws IOException {
     if (!hasKerberosCredentials()) {
       return;
     }
+
+    // Shutdown the background task performing login renewal.
+    if (getKerberosLoginRenewalExecutor().isPresent()) {
+      getKerberosLoginRenewalExecutor().get().shutdownNow();
+    }
+
     HadoopLoginContext login = getLogin();
     String keytabFile = getKeytab();
     if (login == null || keytabFile == null) {

hadoop-common-project/hadoop-common/src/main/resources/core-default.xml

@@ -656,6 +656,14 @@
     </description>
 </property>
 
+<property>
+  <name>hadoop.kerberos.keytab.login.autorenewal.enabled</name>
+  <value>false</value>
+  <description>Used to enable automatic renewal of keytab based kerberos login.
+    By default the automatic renewal is disabled for keytab based kerberos login.
+  </description>
+</property>
+
 <property>
   <name>hadoop.security.auth_to_local</name>
   <value></value>

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGILoginFromKeytab.java

@@ -22,6 +22,7 @@
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.minikdc.MiniKdc;
 import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -31,7 +32,10 @@
 import org.mockito.Mockito;
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
+import org.slf4j.event.Level;
 
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_KERBEROS_KEYTAB_LOGIN_AUTORENEWAL_ENABLED;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -199,6 +203,58 @@ public void testGetUGIFromExternalSubjectWithLogin() throws Exception {
     Assert.assertSame(dummyLogin, user.getLogin());
   }
 
+  @Test
+  public void testUGIRefreshFromKeytab() throws Exception {
+    final Configuration conf = new Configuration();
+    conf.setBoolean(HADOOP_KERBEROS_KEYTAB_LOGIN_AUTORENEWAL_ENABLED, true);
+    SecurityUtil.setAuthenticationMethod(
+            UserGroupInformation.AuthenticationMethod.KERBEROS, conf);
+    UserGroupInformation.setConfiguration(conf);
+
+    String principal = "bar";
+    File keytab = new File(workDir, "bar.keytab");
+    kdc.createPrincipal(keytab, principal);
+
+    UserGroupInformation.loginUserFromKeytab(principal, keytab.getPath());
+
+    UserGroupInformation ugi = UserGroupInformation.getLoginUser();
+
+    Assert.assertEquals(UserGroupInformation.AuthenticationMethod.KERBEROS,
+        ugi.getAuthenticationMethod());
+    Assert.assertTrue(ugi.isFromKeytab());
+    Assert.assertTrue(
+            UserGroupInformation.isKerberosKeyTabLoginRenewalEnabled());
+    Assert.assertTrue(
+            UserGroupInformation.getKerberosLoginRenewalExecutor()
+                    .isPresent());
+  }
+
+  @Test
+  public void testUGIRefreshFromKeytabDisabled() throws Exception {
+    GenericTestUtils.setLogLevel(UserGroupInformation.LOG, Level.DEBUG);
+    final Configuration conf = new Configuration();
+    conf.setLong(HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN, 1);
+    conf.setBoolean(HADOOP_KERBEROS_KEYTAB_LOGIN_AUTORENEWAL_ENABLED, false);
+    SecurityUtil.setAuthenticationMethod(
+            UserGroupInformation.AuthenticationMethod.KERBEROS, conf);
+    UserGroupInformation.setConfiguration(conf);
+
+    String principal = "bar";
+    File keytab = new File(workDir, "bar.keytab");
+    kdc.createPrincipal(keytab, principal);
+
+    UserGroupInformation.loginUserFromKeytab(principal, keytab.getPath());
+
+    UserGroupInformation ugi = UserGroupInformation.getLoginUser();
+    Assert.assertEquals(UserGroupInformation.AuthenticationMethod.KERBEROS,
+            ugi.getAuthenticationMethod());
+    Assert.assertTrue(ugi.isFromKeytab());
+    Assert.assertFalse(
+            UserGroupInformation.isKerberosKeyTabLoginRenewalEnabled());
+    Assert.assertFalse(
+            UserGroupInformation.getKerberosLoginRenewalExecutor()
+                    .isPresent());
+  }
 
   private static KerberosTicket getTicket(UserGroupInformation ugi) {
     Set<KerberosTicket> tickets =

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java

@@ -1239,7 +1239,7 @@ public void testKerberosTicketIsDestroyedChecked() throws Exception {
 
     // run AutoRenewalForUserCredsRunnable with this
     UserGroupInformation.AutoRenewalForUserCredsRunnable userCredsRunnable =
-        ugi.new AutoRenewalForUserCredsRunnable(tgt,
+        ugi.new TicketCacheRenewalRunnable(tgt,
             Boolean.toString(Boolean.TRUE), 100);
 
     // Set the runnable to not to run in a loop