From bfb9adc2b9e6e97f1036bcf8ea4cee6893a782b2 Mon Sep 17 00:00:00 2001 From: Hrishikesh Gadre Date: Sat, 27 Oct 2018 08:58:10 -0700 Subject: [PATCH] HADOOP-9567. Provide auto-renewal for keytab based logins. Contributed by Hrishikesh Gadre, Gary Helmling and Harsh J. Signed-off-by: Wei-Chiu Chuang --- .../fs/CommonConfigurationKeysPublic.java | 12 ++ .../hadoop/security/UserGroupInformation.java | 192 +++++++++++++++--- .../src/main/resources/core-default.xml | 8 + .../security/TestUGILoginFromKeytab.java | 56 +++++ .../security/TestUserGroupInformation.java | 2 +- 5 files changed, 245 insertions(+), 25 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index 852342357e..7410c391ba 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -636,6 +636,18 @@ public class CommonConfigurationKeysPublic { /** Default value for HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN */ public static final int HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN_DEFAULT = 60; + + /** + * @see + * + * core-default.xml + */ + public static final String HADOOP_KERBEROS_KEYTAB_LOGIN_AUTORENEWAL_ENABLED = + "hadoop.kerberos.keytab.login.autorenewal.enabled"; + /** Default value for HADOOP_KERBEROS_KEYTAB_LOGIN_AUTORENEWAL_ENABLED. */ + public static final boolean + HADOOP_KERBEROS_KEYTAB_LOGIN_AUTORENEWAL_ENABLED_DEFAULT = false; + /** * @see * diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java index 915d6df9e5..60a85c18d8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java @@ -20,6 +20,8 @@ import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_USER_GROUP_METRICS_PERCENTILES_INTERVALS; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN_DEFAULT; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_KERBEROS_KEYTAB_LOGIN_AUTORENEWAL_ENABLED; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_KERBEROS_KEYTAB_LOGIN_AUTORENEWAL_ENABLED_DEFAULT; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_TOKEN_FILES; import static org.apache.hadoop.security.UGIExceptionMessages.*; import static org.apache.hadoop.util.PlatformName.IBM_JAVA; @@ -46,7 +48,11 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -280,6 +286,11 @@ public static void reattachMetrics() { private static Groups groups; /** Min time (in seconds) before relogin for Kerberos */ private static long kerberosMinSecondsBeforeRelogin; + /** Boolean flag to enable auto-renewal for keytab based loging. */ + private static boolean kerberosKeyTabLoginRenewalEnabled; + /** A reference to Kerberos login auto renewal thread. */ + private static Optional kerberosLoginRenewalExecutor = + Optional.empty(); /** The configuration to use */ private static Configuration conf; @@ -332,6 +343,11 @@ private static synchronized void initialize(Configuration conf, HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN + " of " + conf.get(HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN)); } + + kerberosKeyTabLoginRenewalEnabled = conf.getBoolean( + HADOOP_KERBEROS_KEYTAB_LOGIN_AUTORENEWAL_ENABLED, + HADOOP_KERBEROS_KEYTAB_LOGIN_AUTORENEWAL_ENABLED_DEFAULT); + // If we haven't set up testing groups, use the configuration to find it if (!(groups instanceof TestingGroups)) { groups = Groups.getUserToGroupsMappingService(conf); @@ -372,6 +388,8 @@ public static void reset() { conf = null; groups = null; kerberosMinSecondsBeforeRelogin = 0; + kerberosKeyTabLoginRenewalEnabled = false; + kerberosLoginRenewalExecutor = Optional.empty(); setLoginUser(null); HadoopKerberosName.setRules(null); } @@ -392,7 +410,23 @@ private static boolean isAuthenticationMethodEnabled(AuthenticationMethod method ensureInitialized(); return (authenticationMethod == method); } - + + @InterfaceAudience.Private + @InterfaceStability.Evolving + @VisibleForTesting + static boolean isKerberosKeyTabLoginRenewalEnabled() { + ensureInitialized(); + return kerberosKeyTabLoginRenewalEnabled; + } + + @InterfaceAudience.Private + @InterfaceStability.Evolving + @VisibleForTesting + static Optional getKerberosLoginRenewalExecutor() { + ensureInitialized(); + return kerberosLoginRenewalExecutor; + } + /** * Information about the logged in user. */ @@ -838,14 +872,16 @@ public boolean shouldRelogin() { return hasKerberosCredentials() && isHadoopLogin(); } + /** + * Spawn a thread to do periodic renewals of kerberos credentials. NEVER + * directly call this method. This method should only be used for ticket cache + * based kerberos credentials. + * + * @param force - used by tests to forcibly spawn thread + */ @InterfaceAudience.Private @InterfaceStability.Unstable @VisibleForTesting - /** - * Spawn a thread to do periodic renewals of kerberos credentials from - * a ticket cache. NEVER directly call this method. - * @param force - used by tests to forcibly spawn thread - */ void spawnAutoRenewalThreadForUserCreds(boolean force) { if (!force && (!shouldRelogin() || isFromKeytab())) { return; @@ -858,25 +894,71 @@ void spawnAutoRenewalThreadForUserCreds(boolean force) { } String cmd = conf.get("hadoop.kerberos.kinit.command", "kinit"); long nextRefresh = getRefreshTime(tgt); - Thread t = - new Thread(new AutoRenewalForUserCredsRunnable(tgt, cmd, nextRefresh)); - t.setDaemon(true); - t.setName("TGT Renewer for " + getUserName()); - t.start(); + executeAutoRenewalTask(getUserName(), + new TicketCacheRenewalRunnable(tgt, cmd, nextRefresh)); } + /** + * Spawn a thread to do periodic renewals of kerberos credentials from a + * keytab file. + */ + private void spawnAutoRenewalThreadForKeytab() { + if (!shouldRelogin() || isFromTicket()) { + return; + } + + // spawn thread only if we have kerb credentials + KerberosTicket tgt = getTGT(); + if (tgt == null) { + return; + } + long nextRefresh = getRefreshTime(tgt); + executeAutoRenewalTask(getUserName(), + new KeytabRenewalRunnable(tgt, nextRefresh)); + } + + /** + * Spawn a thread to do periodic renewals of kerberos credentials from a + * keytab file. NEVER directly call this method. + * + * @param userName Name of the user for which login needs to be renewed. + * @param task The reference of the login renewal task. + */ + private void executeAutoRenewalTask(final String userName, + AutoRenewalForUserCredsRunnable task) { + kerberosLoginRenewalExecutor = Optional.of( + Executors.newSingleThreadExecutor( + new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + Thread t = new Thread(r); + t.setDaemon(true); + t.setName("TGT Renewer for " + userName); + return t; + } + } + )); + kerberosLoginRenewalExecutor.get().submit(task); + } + + /** + * An abstract class which encapsulates the functionality required to + * auto renew Kerbeors TGT. The concrete implementations of this class + * are expected to provide implementation required to perform actual + * TGT renewal (see {@code TicketCacheRenewalRunnable} and + * {@code KeytabRenewalRunnable}). + */ + @InterfaceAudience.Private + @InterfaceStability.Unstable @VisibleForTesting - class AutoRenewalForUserCredsRunnable implements Runnable { + abstract class AutoRenewalForUserCredsRunnable implements Runnable { private KerberosTicket tgt; private RetryPolicy rp; - private String kinitCmd; private long nextRefresh; private boolean runRenewalLoop = true; - AutoRenewalForUserCredsRunnable(KerberosTicket tgt, String kinitCmd, - long nextRefresh){ + AutoRenewalForUserCredsRunnable(KerberosTicket tgt, long nextRefresh) { this.tgt = tgt; - this.kinitCmd = kinitCmd; this.nextRefresh = nextRefresh; this.rp = null; } @@ -885,6 +967,13 @@ public void setRunRenewalLoop(boolean runRenewalLoop) { this.runRenewalLoop = runRenewalLoop; } + /** + * This method is used to perform renewal of kerberos login ticket. + * The concrete implementations of this class should provide specific + * logic required to perform renewal as part of this method. + */ + protected abstract void relogin() throws IOException; + @Override public void run() { do { @@ -897,11 +986,7 @@ public void run() { if (now < nextRefresh) { Thread.sleep(nextRefresh - now); } - String output = Shell.execCommand(kinitCmd, "-R"); - if (LOG.isDebugEnabled()) { - LOG.debug("Renewed ticket. kinit output: {}", output); - } - reloginFromTicketCache(); + relogin(); tgt = getTGT(); if (tgt == null) { LOG.warn("No TGT after renewal. Aborting renew thread for " + @@ -971,6 +1056,52 @@ public void run() { } } + /** + * A concrete implementation of {@code AutoRenewalForUserCredsRunnable} class + * which performs TGT renewal using kinit command. + */ + @InterfaceAudience.Private + @InterfaceStability.Unstable + @VisibleForTesting + final class TicketCacheRenewalRunnable + extends AutoRenewalForUserCredsRunnable { + private String kinitCmd; + + TicketCacheRenewalRunnable(KerberosTicket tgt, String kinitCmd, + long nextRefresh) { + super(tgt, nextRefresh); + this.kinitCmd = kinitCmd; + } + + @Override + public void relogin() throws IOException { + String output = Shell.execCommand(kinitCmd, "-R"); + if (LOG.isDebugEnabled()) { + LOG.debug("Renewed ticket. kinit output: {}", output); + } + reloginFromTicketCache(); + } + } + + /** + * A concrete implementation of {@code AutoRenewalForUserCredsRunnable} class + * which performs TGT renewal using specified keytab. + */ + @InterfaceAudience.Private + @InterfaceStability.Unstable + @VisibleForTesting + final class KeytabRenewalRunnable extends AutoRenewalForUserCredsRunnable { + + KeytabRenewalRunnable(KerberosTicket tgt, long nextRefresh) { + super(tgt, nextRefresh); + } + + @Override + public void relogin() throws IOException { + reloginFromKeytab(); + } + } + /** * Get time for next login retry. This will allow the thread to retry with * exponential back-off, until tgt endtime. @@ -1007,9 +1138,16 @@ static void loginUserFromKeytab(String user, if (!isSecurityEnabled()) return; - setLoginUser(loginUserFromKeytabAndReturnUGI(user, path)); - LOG.info("Login successful for user " + user - + " using keytab file " + path); + UserGroupInformation u = loginUserFromKeytabAndReturnUGI(user, path); + if (isKerberosKeyTabLoginRenewalEnabled()) { + u.spawnAutoRenewalThreadForKeytab(); + } + + setLoginUser(u); + + LOG.info("Login successful for user {} using keytab file {}. Keytab auto" + + " renewal enabled : {}", + user, path, isKerberosKeyTabLoginRenewalEnabled()); } /** @@ -1027,6 +1165,12 @@ public void logoutUserFromKeytab() throws IOException { if (!hasKerberosCredentials()) { return; } + + // Shutdown the background task performing login renewal. + if (getKerberosLoginRenewalExecutor().isPresent()) { + getKerberosLoginRenewalExecutor().get().shutdownNow(); + } + HadoopLoginContext login = getLogin(); String keytabFile = getKeytab(); if (login == null || keytabFile == null) { diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index ce3a407960..b243a9c63c 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -656,6 +656,14 @@ + + hadoop.kerberos.keytab.login.autorenewal.enabled + false + Used to enable automatic renewal of keytab based kerberos login. + By default the automatic renewal is disabled for keytab based kerberos login. + + + hadoop.security.auth_to_local diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGILoginFromKeytab.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGILoginFromKeytab.java index 826e4b2ea0..8ede451db9 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGILoginFromKeytab.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGILoginFromKeytab.java @@ -22,6 +22,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.minikdc.MiniKdc; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -31,7 +32,10 @@ import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; +import org.slf4j.event.Level; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_KERBEROS_KEYTAB_LOGIN_AUTORENEWAL_ENABLED; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -199,6 +203,58 @@ public void testGetUGIFromExternalSubjectWithLogin() throws Exception { Assert.assertSame(dummyLogin, user.getLogin()); } + @Test + public void testUGIRefreshFromKeytab() throws Exception { + final Configuration conf = new Configuration(); + conf.setBoolean(HADOOP_KERBEROS_KEYTAB_LOGIN_AUTORENEWAL_ENABLED, true); + SecurityUtil.setAuthenticationMethod( + UserGroupInformation.AuthenticationMethod.KERBEROS, conf); + UserGroupInformation.setConfiguration(conf); + + String principal = "bar"; + File keytab = new File(workDir, "bar.keytab"); + kdc.createPrincipal(keytab, principal); + + UserGroupInformation.loginUserFromKeytab(principal, keytab.getPath()); + + UserGroupInformation ugi = UserGroupInformation.getLoginUser(); + + Assert.assertEquals(UserGroupInformation.AuthenticationMethod.KERBEROS, + ugi.getAuthenticationMethod()); + Assert.assertTrue(ugi.isFromKeytab()); + Assert.assertTrue( + UserGroupInformation.isKerberosKeyTabLoginRenewalEnabled()); + Assert.assertTrue( + UserGroupInformation.getKerberosLoginRenewalExecutor() + .isPresent()); + } + + @Test + public void testUGIRefreshFromKeytabDisabled() throws Exception { + GenericTestUtils.setLogLevel(UserGroupInformation.LOG, Level.DEBUG); + final Configuration conf = new Configuration(); + conf.setLong(HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN, 1); + conf.setBoolean(HADOOP_KERBEROS_KEYTAB_LOGIN_AUTORENEWAL_ENABLED, false); + SecurityUtil.setAuthenticationMethod( + UserGroupInformation.AuthenticationMethod.KERBEROS, conf); + UserGroupInformation.setConfiguration(conf); + + String principal = "bar"; + File keytab = new File(workDir, "bar.keytab"); + kdc.createPrincipal(keytab, principal); + + UserGroupInformation.loginUserFromKeytab(principal, keytab.getPath()); + + UserGroupInformation ugi = UserGroupInformation.getLoginUser(); + Assert.assertEquals(UserGroupInformation.AuthenticationMethod.KERBEROS, + ugi.getAuthenticationMethod()); + Assert.assertTrue(ugi.isFromKeytab()); + Assert.assertFalse( + UserGroupInformation.isKerberosKeyTabLoginRenewalEnabled()); + Assert.assertFalse( + UserGroupInformation.getKerberosLoginRenewalExecutor() + .isPresent()); + } private static KerberosTicket getTicket(UserGroupInformation ugi) { Set tickets = diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java index 011e930e50..3020f9be14 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java @@ -1239,7 +1239,7 @@ public void testKerberosTicketIsDestroyedChecked() throws Exception { // run AutoRenewalForUserCredsRunnable with this UserGroupInformation.AutoRenewalForUserCredsRunnable userCredsRunnable = - ugi.new AutoRenewalForUserCredsRunnable(tgt, + ugi.new TicketCacheRenewalRunnable(tgt, Boolean.toString(Boolean.TRUE), 100); // Set the runnable to not to run in a loop