From d43af8b3db4743b4b240751b6f29de6c20cfd6e5 Mon Sep 17 00:00:00 2001 From: Eric Yang Date: Fri, 4 Jan 2019 17:48:18 -0500 Subject: [PATCH] HADOOP-15996. Improved Kerberos username mapping strategy in Hadoop. Contributed by Bolke de Bruin --- .../server/KerberosAuthenticationHandler.java | 11 ++- .../authentication/util/KerberosName.java | 87 ++++++++++++++++--- .../client/TestKerberosAuthenticator.java | 1 + .../TestKerberosAuthenticationHandler.java | 59 +++++++++++-- .../authentication/util/TestKerberosName.java | 12 +++ .../fs/CommonConfigurationKeysPublic.java | 7 ++ .../org/apache/hadoop/http/HttpServer2.java | 1 - .../hadoop/security/HadoopKerberosName.java | 11 ++- .../org/apache/hadoop/security/KDiag.java | 45 +++++++++- .../src/main/resources/core-default.xml | 9 ++ .../src/site/markdown/SecureMode.md | 29 +++++-- .../org/apache/hadoop/security/TestKDiag.java | 16 ++++ .../security/TestUserGroupInformation.java | 36 +++++++- 13 files changed, 295 insertions(+), 29 deletions(-) diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java index 887548ba85..7a918b20bb 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/KerberosAuthenticationHandler.java @@ -88,6 +88,12 @@ public class KerberosAuthenticationHandler implements AuthenticationHandler { */ public static final String NAME_RULES = TYPE + ".name.rules"; + /** + * Constant for the configuration property that indicates how auth_to_local + * rules are evaluated. + */ + public static final String RULE_MECHANISM = TYPE + ".name.rules.mechanism"; + private String type; private String keytab; private GSSManager gssManager; @@ -163,7 +169,10 @@ public void init(Properties config) throws ServletException { if (nameRules != null) { KerberosName.setRules(nameRules); } - + String ruleMechanism = config.getProperty(RULE_MECHANISM, null); + if (ruleMechanism != null) { + KerberosName.setRuleMechanism(ruleMechanism); + } try { gssManager = Subject.doAs(serverSubject, new PrivilegedExceptionAction() { diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java index 287bb13917..684d2c8c1e 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosName.java @@ -44,6 +44,19 @@ public class KerberosName { private static final Logger LOG = LoggerFactory.getLogger(KerberosName.class); + /** + * Constant that defines auth_to_local legacy hadoop evaluation + */ + public static final String MECHANISM_HADOOP = "hadoop"; + + /** + * Constant that defines auth_to_local MIT evaluation + */ + public static final String MECHANISM_MIT = "mit"; + + /** Constant that defines the default behavior of the rule mechanism */ + public static final String DEFAULT_MECHANISM = MECHANISM_HADOOP; + /** The first component of the name */ private final String serviceName; /** The second component of the name. It may be null. */ @@ -81,6 +94,11 @@ public class KerberosName { */ private static List rules; + /** + * How to evaluate auth_to_local rules + */ + private static String ruleMechanism = null; + private static String defaultRealm = null; @VisibleForTesting @@ -304,10 +322,11 @@ static String replaceSubstitution(String base, Pattern from, String to, * array. * @param params first element is the realm, second and later elements are * are the components of the name "a/b@FOO" -> {"FOO", "a", "b"} + * @param ruleMechanism defines the rule evaluation mechanism * @return the short name if this rule applies or null * @throws IOException throws if something is wrong with the rules */ - String apply(String[] params) throws IOException { + String apply(String[] params, String ruleMechanism) throws IOException { String result = null; if (isDefault) { if (getDefaultRealm().equals(params[0])) { @@ -323,7 +342,9 @@ String apply(String[] params) throws IOException { } } } - if (result != null && nonSimplePattern.matcher(result).find()) { + if (result != null + && nonSimplePattern.matcher(result).find() + && ruleMechanism.equalsIgnoreCase(MECHANISM_HADOOP)) { throw new NoMatchingRule("Non-simple name " + result + " after auth_to_local rule " + this); } @@ -392,21 +413,22 @@ public String getShortName() throws IOException { } else { params = new String[]{realm, serviceName, hostName}; } + String ruleMechanism = this.ruleMechanism; + if (ruleMechanism == null && rules != null) { + LOG.warn("auth_to_local rule mechanism not set." + + "Using default of " + DEFAULT_MECHANISM); + ruleMechanism = DEFAULT_MECHANISM; + } for(Rule r: rules) { - String result = r.apply(params); + String result = r.apply(params, ruleMechanism); if (result != null) { return result; } } - throw new NoMatchingRule("No rules applied to " + toString()); - } - - /** - * Set the rules. - * @param ruleString the rules string. - */ - public static void setRules(String ruleString) { - rules = (ruleString != null) ? parseRules(ruleString) : null; + if (ruleMechanism.equalsIgnoreCase(MECHANISM_HADOOP)) { + throw new NoMatchingRule("No rules applied to " + toString()); + } + return toString(); } /** @@ -434,6 +456,47 @@ public static boolean hasRulesBeenSet() { return rules != null; } + /** + * Indicates of the rule mechanism has been set + * + * @return if the rule mechanism has been set. + */ + public static boolean hasRuleMechanismBeenSet() { + return ruleMechanism != null; + } + + /** + * Set the rules. + * @param ruleString the rules string. + */ + public static void setRules(String ruleString) { + rules = (ruleString != null) ? parseRules(ruleString) : null; + } + + /** + * + * @param ruleMech the evaluation type: hadoop, mit + * 'hadoop' indicates '@' or '/' are not allowed the result + * evaluation. 'MIT' indicates that auth_to_local + * rules follow MIT Kerberos evaluation. + */ + public static void setRuleMechanism(String ruleMech) { + if (ruleMech != null + && (!ruleMech.equalsIgnoreCase(MECHANISM_HADOOP) + && !ruleMech.equalsIgnoreCase(MECHANISM_MIT))) { + throw new IllegalArgumentException("Invalid rule mechanism: " + ruleMech); + } + ruleMechanism = ruleMech; + } + + /** + * Get the rule evaluation mechanism + * @return the rule evaluation mechanism + */ + public static String getRuleMechanism() { + return ruleMechanism; + } + static void printRules() throws IOException { int i = 0; for(Rule r: rules) { diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestKerberosAuthenticator.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestKerberosAuthenticator.java index 4aabb34fa5..177bcb4547 100644 --- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestKerberosAuthenticator.java +++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestKerberosAuthenticator.java @@ -65,6 +65,7 @@ private Properties getAuthenticationHandlerConfiguration() { props.setProperty(KerberosAuthenticationHandler.KEYTAB, KerberosTestUtils.getKeytabFile()); props.setProperty(KerberosAuthenticationHandler.NAME_RULES, "RULE:[1:$1@$0](.*@" + KerberosTestUtils.getRealm()+")s/@.*//\n"); + props.setProperty(KerberosAuthenticationHandler.RULE_MECHANISM, "hadoop"); return props; } diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java index d0709bf4ca..bef3018ca4 100644 --- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java +++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestKerberosAuthenticationHandler.java @@ -70,6 +70,8 @@ protected Properties getDefaultProperties() { KerberosTestUtils.getKeytabFile()); props.setProperty(KerberosAuthenticationHandler.NAME_RULES, "RULE:[1:$1@$0](.*@" + KerberosTestUtils.getRealm()+")s/@.*//\n"); + props.setProperty(KerberosAuthenticationHandler.RULE_MECHANISM, + KerberosName.MECHANISM_HADOOP); return props; } @@ -96,19 +98,18 @@ public void setup() throws Exception { } @Test - public void testNameRules() throws Exception { + public void testNameRulesHadoop() throws Exception { KerberosName kn = new KerberosName(KerberosTestUtils.getServerPrincipal()); Assert.assertEquals(KerberosTestUtils.getRealm(), kn.getRealm()); //destroy handler created in setUp() handler.destroy(); - - KerberosName.setRules("RULE:[1:$1@$0](.*@FOO)s/@.*//\nDEFAULT"); - handler = getNewAuthenticationHandler(); + Properties props = getDefaultProperties(); props.setProperty(KerberosAuthenticationHandler.NAME_RULES, "RULE:[1:$1@$0](.*@BAR)s/@.*//\nDEFAULT"); + try { handler.init(props); } catch (Exception ex) { @@ -124,7 +125,55 @@ public void testNameRules() throws Exception { } @Test - public void testInit() { + public void testNameRulesCompat() throws Exception { + KerberosName kn = new KerberosName(KerberosTestUtils.getServerPrincipal()); + Assert.assertEquals(KerberosTestUtils.getRealm(), kn.getRealm()); + + //destroy handler created in setUp() + handler.destroy(); + handler = getNewAuthenticationHandler(); + + Properties props = getDefaultProperties(); + props.setProperty(KerberosAuthenticationHandler.NAME_RULES, "RULE:[1:$1@$0](.*@BAR)s/@.*//\nDEFAULT"); + props.setProperty(KerberosAuthenticationHandler.RULE_MECHANISM, KerberosName.MECHANISM_MIT); + + try { + handler.init(props); + } catch (Exception ex) { + } + kn = new KerberosName("bar@BAR"); + Assert.assertEquals("bar", kn.getShortName()); + kn = new KerberosName("bar@FOO"); + Assert.assertEquals("bar@FOO", kn.getShortName()); + } + + @Test + public void testNullProperties() throws Exception { + KerberosName kn = new KerberosName(KerberosTestUtils.getServerPrincipal()); + Assert.assertEquals(KerberosTestUtils.getRealm(), kn.getRealm()); + + KerberosName.setRuleMechanism("MIT"); + KerberosName.setRules("DEFAULT"); + + //destroy handler created in setUp() + handler.destroy(); + handler = getNewAuthenticationHandler(); + + Properties props = getDefaultProperties(); + props.remove(KerberosAuthenticationHandler.NAME_RULES); + props.remove(KerberosAuthenticationHandler.RULE_MECHANISM); + + try { + handler.init(props); + } catch (Exception ex) { + } + + Assert.assertEquals("MIT", KerberosName.getRuleMechanism()); + Assert.assertEquals("DEFAULT", KerberosName.getRules()); + } + + @Test + public void testInit() throws Exception { Assert.assertEquals(KerberosTestUtils.getKeytabFile(), handler.getKeytab()); Set principals = handler.getPrincipals(); Principal expectedPrincipal = diff --git a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java index 2db0df4d54..105fa11424 100644 --- a/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java +++ b/hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestKerberosName.java @@ -40,6 +40,7 @@ public void setUp() throws Exception { "RULE:[2:$1;$2](^.*;admin$)s/;admin$//\n" + "RULE:[2:$2](root)\n" + "DEFAULT"; + KerberosName.setRuleMechanism(KerberosName.MECHANISM_HADOOP); KerberosName.setRules(rules); KerberosName.printRules(); } @@ -85,10 +86,16 @@ private void checkBadTranslation(String from) { @Test public void testAntiPatterns() throws Exception { + KerberosName.setRuleMechanism(KerberosName.MECHANISM_HADOOP); checkBadName("owen/owen/owen@FOO.COM"); checkBadName("owen@foo/bar.com"); + checkBadTranslation("foo@ACME.COM"); checkBadTranslation("root/joe@FOO.COM"); + + KerberosName.setRuleMechanism(KerberosName.MECHANISM_MIT); + checkTranslation("foo@ACME.COM", "foo@ACME.COM"); + checkTranslation("root/joe@FOO.COM", "root/joe@FOO.COM"); } @Test @@ -129,6 +136,11 @@ public void testToLowerCase() throws Exception { checkTranslation("Joe/guestguest@FOO.COM", "joe"); } + @Test(expected = IllegalArgumentException.class) + public void testInvalidRuleMechanism() throws Exception { + KerberosName.setRuleMechanism("INVALID_MECHANISM"); + } + @After public void clear() { System.clearProperty("java.security.krb5.realm"); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index 7410c391ba..5f3426560a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -607,6 +607,13 @@ public class CommonConfigurationKeysPublic { * * core-default.xml */ + public static final String HADOOP_SECURITY_AUTH_TO_LOCAL_MECHANISM = + "hadoop.security.auth_to_local.mechanism"; + /** + * @see + * + * core-default.xml + */ public static final String HADOOP_SECURITY_DNS_INTERFACE_KEY = "hadoop.security.dns.interface"; /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java index 598d3ee146..fb2dff5d02 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java @@ -1129,7 +1129,6 @@ private void initSpnego(Configuration conf, String hostName, params.put("kerberos.keytab", httpKeytab); } params.put(AuthenticationFilter.AUTH_TYPE, "kerberos"); - defineFilter(webAppContext, SPNEGO_FILTER, AuthenticationFilter.class.getName(), params, null); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/HadoopKerberosName.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/HadoopKerberosName.java index d7f0e81640..df96c500cd 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/HadoopKerberosName.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/HadoopKerberosName.java @@ -19,6 +19,7 @@ package org.apache.hadoop.security; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL_MECHANISM; import java.io.IOException; @@ -27,6 +28,9 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.authentication.util.KerberosName; import org.apache.hadoop.security.authentication.util.KerberosUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * This class implements parsing and handling of Kerberos principal names. In * particular, it splits them apart and translates them down into local @@ -36,6 +40,8 @@ @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"}) @InterfaceStability.Evolving public class HadoopKerberosName extends KerberosName { + private static final Logger LOG = + LoggerFactory.getLogger(HadoopKerberosName.class); /** * Create a name from the full Kerberos principal name. @@ -45,7 +51,7 @@ public HadoopKerberosName(String name) { super(name); } /** - * Set the static configuration to get the rules. + * Set the static configuration to get and evaluate the rules. *

* IMPORTANT: This method does a NOP if the rules have been set already. * If there is a need to reset the rules, the {@link KerberosName#setRules(String)} @@ -73,6 +79,9 @@ public static void setConfiguration(Configuration conf) throws IOException { } String ruleString = conf.get(HADOOP_SECURITY_AUTH_TO_LOCAL, defaultRule); setRules(ruleString); + + String ruleMechanism = conf.get(HADOOP_SECURITY_AUTH_TO_LOCAL_MECHANISM, DEFAULT_MECHANISM); + setRuleMechanism(ruleMechanism); } public static void main(String[] args) throws Exception { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java index 17119213de..2c6de4d216 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java @@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.authentication.util.KerberosName; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.util.ExitUtil; @@ -54,6 +55,7 @@ import java.util.Date; import java.util.LinkedList; import java.util.List; +import java.util.regex.Pattern; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*; import static org.apache.hadoop.security.UserGroupInformation.*; @@ -129,6 +131,12 @@ public class KDiag extends Configured implements Tool, Closeable { private boolean nofail = false; private boolean nologin = false; private boolean jaas = false; + private boolean checkShortName = false; + + /** + * A pattern that recognizes simple/non-simple names. Per KerberosName + */ + private static final Pattern nonSimplePattern = Pattern.compile("[/@]"); /** * Flag set to true if a {@link #verify(boolean, String, String, Object...)} @@ -157,6 +165,8 @@ public class KDiag extends Configured implements Tool, Closeable { public static final String ARG_SECURE = "--secure"; + public static final String ARG_VERIFYSHORTNAME = "--verifyshortname"; + @SuppressWarnings("IOResourceOpenedButNotSafelyClosed") public KDiag(Configuration conf, PrintWriter out, @@ -200,6 +210,7 @@ public int run(String[] argv) throws Exception { nofail = popOption(ARG_NOFAIL, args); jaas = popOption(ARG_JAAS, args); nologin = popOption(ARG_NOLOGIN, args); + checkShortName = popOption(ARG_VERIFYSHORTNAME, args); // look for list of resources String resource; @@ -245,7 +256,9 @@ private String usage() { + arg(ARG_NOLOGIN, "", "Do not attempt to log in") + arg(ARG_OUTPUT, "", "Write output to a file") + arg(ARG_RESOURCE, "", "Load an XML configuration resource") - + arg(ARG_SECURE, "", "Require the hadoop configuration to be secure"); + + arg(ARG_SECURE, "", "Require the hadoop configuration to be secure") + + arg(ARG_VERIFYSHORTNAME, ARG_PRINCIPAL + " ", + "Verify the short name of the specific principal does not contain '@' or '/'"); } private String arg(String name, String params, String meaning) { @@ -278,6 +291,7 @@ public boolean execute() throws Exception { println("%s = %d", ARG_KEYLEN, minKeyLength); println("%s = %s", ARG_KEYTAB, keytab); println("%s = %s", ARG_PRINCIPAL, principal); + println("%s = %s", ARG_VERIFYSHORTNAME, checkShortName); // Fail fast on a JVM without JCE installed. validateKeyLength(); @@ -376,6 +390,9 @@ public boolean execute() throws Exception { validateKinitExecutable(); validateJAAS(jaas); validateNTPConf(); + if (checkShortName) { + validateShortName(); + } if (!nologin) { title("Logging in"); @@ -430,6 +447,32 @@ protected void validateKeyLength() throws NoSuchAlgorithmException { aesLen, minKeyLength); } + /** + * Verify whether auth_to_local rules transform a principal name + *

+ * Having a local user name "bar@foo.com" may be harmless, so it is noted at + * info. However if what was intended is a transformation to "bar" + * it can be difficult to debug, hence this check. + */ + protected void validateShortName() { + failif(principal == null, CAT_KERBEROS, "No principal defined"); + + try { + KerberosName kn = new KerberosName(principal); + String result = kn.getShortName(); + if (nonSimplePattern.matcher(result).find()) { + warn(CAT_KERBEROS, principal + " short name: " + result + + " still contains @ or /"); + } + } catch (IOException e) { + throw new KerberosDiagsFailure(CAT_KERBEROS, e, + "Failed to get short name for " + principal, e); + } catch (IllegalArgumentException e) { + error(CAT_KERBEROS, "KerberosName(" + principal + ") failed: %s\n%s", + e, StringUtils.stringifyException(e)); + } + } + /** * Get the default realm. *

diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index ddcee2f7b5..bda20105af 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -704,6 +704,15 @@ Maps kerberos principals to local user names + + hadoop.security.auth_to_local.mechanism + hadoop + The mechanism by which auth_to_local rules are evaluated. + If set to 'hadoop' it will not allow resulting local user names to have + either '@' or '/'. If set to 'MIT' it will follow MIT evaluation rules + and the restrictions of 'hadoop' do not apply. + + hadoop.token.files diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md b/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md index 09f1eed892..856861f29e 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md @@ -133,22 +133,35 @@ The MapReduce JobHistory Server keytab file, on that host, should look like the ### Mapping from Kerberos principals to OS user accounts -Hadoop maps Kerberos principals to OS user (system) accounts using rules specified by `hadoop.security.auth_to_local`. These rules work in the same way as the `auth_to_local` in [Kerberos configuration file (krb5.conf)](http://web.mit.edu/Kerberos/krb5-latest/doc/admin/conf_files/krb5_conf.html). In addition, Hadoop `auth_to_local` mapping supports the **/L** flag that lowercases the returned name. +Hadoop maps Kerberos principals to OS user (system) accounts using rules specified by `hadoop.security.auth_to_local`. How Hadoop evaluates these rules is determined by the setting of `hadoop.security.auth_to_local.mechanism`. -The default is to pick the first component of the principal name as the system user name if the realm matches the `default_realm` (usually defined in /etc/krb5.conf). e.g. The default rule maps the principal `host/full.qualified.domain.name@REALM.TLD` to system user `host`. The default rule will *not be appropriate* for most clusters. +In the default `hadoop` mode a Kerberos principal *must* be matched against a rule that transforms the principal to a simple form, i.e. a user account name without '@' or '/', otherwise a principal will not be authorized and a error will be logged. In case of the `MIT` mode the rules work in the same way as the `auth_to_local` in [Kerberos configuration file (krb5.conf)](http://web.mit.edu/Kerberos/krb5-latest/doc/admin/conf_files/krb5_conf.html) and the restrictions of `hadoop` mode do *not* apply. If you use `MIT` mode it is suggested to use the same `auth_to_local` rules that are specified in your /etc/krb5.conf as part of your default realm and keep them in sync. In both `hadoop` and `MIT` mode the rules are being applied (with the exception of `DEFAULT`) to *all* principals regardless of their specified realm. Also, note you should *not* rely on the `auth_to_local` rules as an ACL and use proper (OS) mechanisms. +Possible values for `auth_to_local` are: + +* `RULE:exp` The local name will be formulated from exp. The format for exp is `[n:string](regexp)s/pattern/replacement/g`. The integer n indicates how many components the target principal should have. If this matches, then a string will be formed from string, substituting the realm of the principal for `$0` and the n’th component of the principal for `$n` (e.g., if the principal was johndoe/admin then `[2:$2$1foo]` would result in the string `adminjohndoefoo`). If this string matches regexp, then the `s//[g]` substitution command will be run over the string. The optional g will cause the substitution to be global over the string, instead of replacing only the first match in the string. As an extension to MIT, Hadoop `auth_to_local` mapping supports the **/L** flag that lowercases the returned name. + +* `DEFAULT` Picks the first component of the principal name as the system user name if and only if the realm matches the `default_realm` (usually defined in /etc/krb5.conf). e.g. The default rule maps the principal `host/full.qualified.domain.name@MYREALM.TLD` to system user `host` if the default realm is `MYREALM.TLD`. + +In case no rules are specified Hadoop defaults to using `DEFAULT`, which is probably *not suitable* to most of the clusters. + +Please note that Hadoop does not support multiple default realms (e.g like Heimdal does). Also, Hadoop does not do a verification on mapping whether a local system account exists. + +### Example rules In a typical cluster HDFS and YARN services will be launched as the system `hdfs` and `yarn` users respectively. `hadoop.security.auth_to_local` can be configured as follows: hadoop.security.auth_to_local - RULE:[2:$1/$2@$0]([ndj]n/.*@REALM.TLD)s/.*/hdfs/ - RULE:[2:$1/$2@$0]([rn]m/.*@REALM.TLD)s/.*/yarn/ - RULE:[2:$1/$2@$0](jhs/.*@REALM.TLD)s/.*/mapred/ + RULE:[2:$1/$2@$0]([ndj]n/.*@REALM.\TLD)s/.*/hdfs/ + RULE:[2:$1/$2@$0]([rn]m/.*@REALM\.TLD)s/.*/yarn/ + RULE:[2:$1/$2@$0](jhs/.*@REALM\.TLD)s/.*/mapred/ DEFAULT +This would map any principal `nn, dn, jn` on any `host` from realm `REALM.TLD` to the local system account `hdfs`. Secondly it would map any principal `rm, nm` on any `host` from `REALM.TLD` to the local system account `yarn`. Thirdly, it would map the principal `jhs` on any `host` from realm `REALM.TLD` to the local system account `mapred`. Finally, any principal on any host from the default realm will be mapped to the user component of that principal. + Custom rules can be tested using the `hadoop kerbname` command. This command allows one to specify a principal and apply Hadoop's current `auth_to_local` ruleset. ### Mapping from user to group @@ -470,6 +483,7 @@ KDiag: Diagnose Kerberos Problems [--out ] : Write output to a file. [--resource ] : Load an XML configuration resource. [--secure] : Require the hadoop configuration to be secure. + [--verifyshortname ]: Verify the short name of the specific principal does not contain '@' or '/' ``` #### `--jaas`: Require a JAAS file to be defined in `java.security.auth.login.config`. @@ -565,6 +579,11 @@ or implicitly set to "simple": Needless to say, an application so configured cannot talk to a secure Hadoop cluster. +#### `--verifyshortname `: validate the short name of a principal + +This verifies that the short name of a principal contains neither the `"@"` +nor `"/"` characters. + ### Example ``` diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiag.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiag.java index 3895ae11ec..e395566dae 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiag.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiag.java @@ -164,6 +164,22 @@ public void testKeytabAndPrincipal() throws Throwable { ARG_PRINCIPAL, "foo@EXAMPLE.COM"); } + @Test + public void testKerberosName() throws Throwable { + kdiagFailure(ARG_KEYLEN, KEYLEN, + ARG_VERIFYSHORTNAME, + ARG_PRINCIPAL, "foo/foo/foo@BAR.COM"); + } + + @Test + public void testShortName() throws Throwable { + kdiag(ARG_KEYLEN, KEYLEN, + ARG_KEYTAB, keytab.getAbsolutePath(), + ARG_PRINCIPAL, + ARG_VERIFYSHORTNAME, + ARG_PRINCIPAL, "foo@EXAMPLE.COM"); + } + @Test public void testFileOutput() throws Throwable { File f = new File("target/kdiag.txt"); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java index 8af032830f..f41cfc4144 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java @@ -74,6 +74,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_USER_GROUP_METRICS_PERCENTILES_INTERVALS; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL_MECHANISM; import static org.apache.hadoop.test.MetricsAsserts.assertCounter; import static org.apache.hadoop.test.MetricsAsserts.assertCounterGt; import static org.apache.hadoop.test.MetricsAsserts.assertGaugeGt; @@ -329,6 +330,7 @@ public void testConstructorWithRules() throws Exception { // security off, but use rules if explicitly set conf.set(HADOOP_SECURITY_AUTH_TO_LOCAL, "RULE:[1:$1@$0](.*@OTHER.REALM)s/(.*)@.*/other-$1/"); + conf.set(HADOOP_SECURITY_AUTH_TO_LOCAL_MECHANISM, "hadoop"); UserGroupInformation.setConfiguration(conf); testConstructorSuccess("user1", "user1"); testConstructorSuccess("user4@OTHER.REALM", "other-user4"); @@ -336,25 +338,52 @@ public void testConstructorWithRules() throws Exception { testConstructorFailures("user2@DEFAULT.REALM"); testConstructorFailures("user3/cron@DEFAULT.REALM"); testConstructorFailures("user5/cron@OTHER.REALM"); + + // with MIT + conf.set(HADOOP_SECURITY_AUTH_TO_LOCAL_MECHANISM, "mit"); + UserGroupInformation.setConfiguration(conf); + testConstructorSuccess("user2@DEFAULT.REALM", "user2@DEFAULT.REALM"); + testConstructorSuccess("user3/cron@DEFAULT.REALM", "user3/cron@DEFAULT.REALM"); + testConstructorSuccess("user5/cron@OTHER.REALM", "user5/cron@OTHER.REALM"); + + // failures + testConstructorFailures("user6@example.com@OTHER.REALM"); + testConstructorFailures("user7@example.com@DEFAULT.REALM"); testConstructorFailures(null); testConstructorFailures(""); + + conf.set(HADOOP_SECURITY_AUTH_TO_LOCAL_MECHANISM, "hadoop"); + } /** test constructor */ @Test (timeout = 30000) public void testConstructorWithKerberos() throws Exception { // security on, default is remove default realm + conf.set(HADOOP_SECURITY_AUTH_TO_LOCAL_MECHANISM, "hadoop"); SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf); UserGroupInformation.setConfiguration(conf); testConstructorSuccess("user1", "user1"); testConstructorSuccess("user2@DEFAULT.REALM", "user2"); - testConstructorSuccess("user3/cron@DEFAULT.REALM", "user3"); + testConstructorSuccess("user3/cron@DEFAULT.REALM", "user3"); + // failure test testConstructorFailures("user4@OTHER.REALM"); testConstructorFailures("user5/cron@OTHER.REALM"); + + // with MIT + conf.set(HADOOP_SECURITY_AUTH_TO_LOCAL_MECHANISM, "mit"); + UserGroupInformation.setConfiguration(conf); + testConstructorSuccess("user4@OTHER.REALM", "user4@OTHER.REALM"); + testConstructorSuccess("user5/cron@OTHER.REALM", "user5/cron@OTHER.REALM"); + + // failures testConstructorFailures(null); testConstructorFailures(""); + + conf.set(HADOOP_SECURITY_AUTH_TO_LOCAL_MECHANISM, "hadoop"); + } /** test constructor */ @@ -393,8 +422,9 @@ private void testConstructorFailures(String userName) { } catch (IllegalArgumentException e) { String expect = (userName == null || userName.isEmpty()) ? "Null user" : "Illegal principal name "+userName; - assertTrue("Did not find "+ expect + " in " + e, - e.toString().contains(expect)); + String expect2 = "Malformed Kerberos name: "+userName; + assertTrue("Did not find "+ expect + " or " + expect2 + " in " + e, + e.toString().contains(expect) || e.toString().contains(expect2)); } }