From 2664248797365761089a86d5bd59aa9ac3ebcc28 Mon Sep 17 00:00:00 2001 From: Eric Yang Date: Sat, 10 Nov 2018 01:52:19 -0500 Subject: [PATCH] YARN-9002. Improve keytab loading for YARN Service. Contributed by Gour Saha --- .../yarn/service/client/ServiceClient.java | 38 +++++++------------ .../exceptions/RestApiErrorMessages.java | 2 - .../yarn/service/utils/ServiceApiUtil.java | 17 --------- .../service/utils/TestServiceApiUtil.java | 25 +++--------- 4 files changed, 19 insertions(+), 63 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index 91d6367990..1158e446ad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -1392,31 +1392,21 @@ private void addKeytabResourceIfSecure(SliderFileSystem fileSystem, throw new YarnException(e); } - if (keytabURI.getScheme() != null) { - switch (keytabURI.getScheme()) { - case "hdfs": - Path keytabOnhdfs = new Path(keytabURI); - if (!fileSystem.getFileSystem().exists(keytabOnhdfs)) { - LOG.warn(service.getName() + "'s keytab (principalName = " - + principalName + ") doesn't exist at: " + keytabOnhdfs); - return; - } - LocalResource keytabRes = fileSystem.createAmResource(keytabOnhdfs, - LocalResourceType.FILE); - localResource.put(String.format(YarnServiceConstants.KEYTAB_LOCATION, - service.getName()), keytabRes); - LOG.info("Adding " + service.getName() + "'s keytab for " - + "localization, uri = " + keytabOnhdfs); - break; - case "file": - LOG.info("Using a keytab from localhost: " + keytabURI); - break; - default: - LOG.warn("Unsupported keytab URI scheme " + keytabURI); - break; - } + if ("file".equals(keytabURI.getScheme())) { + LOG.info("Using a keytab from localhost: " + keytabURI); } else { - LOG.warn("Unsupported keytab URI scheme " + keytabURI); + Path keytabOnhdfs = new Path(keytabURI); + if (!fileSystem.getFileSystem().exists(keytabOnhdfs)) { + LOG.warn(service.getName() + "'s keytab (principalName = " + + principalName + ") doesn't exist at: " + keytabOnhdfs); + return; + } + LocalResource keytabRes = fileSystem.createAmResource(keytabOnhdfs, + LocalResourceType.FILE); + localResource.put(String.format(YarnServiceConstants.KEYTAB_LOCATION, + service.getName()), keytabRes); + LOG.info("Adding " + service.getName() + "'s keytab for " + + "localization, uri = " + keytabOnhdfs); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java index 8f831ee5a7..57c6449a98 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java @@ -117,8 +117,6 @@ public interface RestApiErrorMessages { + "expression element name %s specified in placement policy of component " + "%s. Expression element names should be a valid constraint name or an " + "expression name defined for this component only."; - String ERROR_KEYTAB_URI_SCHEME_INVALID = "Unsupported keytab URI scheme: %s"; - String ERROR_KEYTAB_URI_INVALID = "Invalid keytab URI: %s"; String ERROR_COMP_INSTANCE_DOES_NOT_NEED_UPGRADE = "The component instance " + "(%s) does not need an upgrade."; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java index b57e6323f2..27e8cafe50 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java @@ -56,8 +56,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; @@ -256,21 +254,6 @@ public static void validateKerberosPrincipal( kerberosPrincipal.getPrincipalName())); } } - if (!StringUtils.isEmpty(kerberosPrincipal.getKeytab())) { - try { - // validate URI format - URI keytabURI = new URI(kerberosPrincipal.getKeytab()); - if (keytabURI.getScheme() == null) { - throw new IllegalArgumentException(String.format( - RestApiErrorMessages.ERROR_KEYTAB_URI_SCHEME_INVALID, - kerberosPrincipal.getKeytab())); - } - } catch (URISyntaxException e) { - throw new IllegalArgumentException( - String.format(RestApiErrorMessages.ERROR_KEYTAB_URI_INVALID, - e.getLocalizedMessage())); - } - } } private static void validateDockerClientConfiguration(Service service, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestServiceApiUtil.java index 4940f8bf29..2bf59b8afc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestServiceApiUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestServiceApiUtil.java @@ -49,7 +49,6 @@ import static org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages.*; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; /** * Test for ServiceApiUtil helper methods. @@ -593,33 +592,19 @@ public void testKerberosPrincipal() throws IOException { SliderFileSystem sfs = ServiceTestUtils.initMockFs(); Service app = createValidApplication("comp-a"); KerberosPrincipal kp = new KerberosPrincipal(); - kp.setKeytab("/some/path"); + kp.setKeytab("file:///tmp/a.keytab"); kp.setPrincipalName("user/_HOST@domain.com"); app.setKerberosPrincipal(kp); + // This should succeed try { ServiceApiUtil.validateKerberosPrincipal(app.getKerberosPrincipal()); - Assert.fail(EXCEPTION_PREFIX + "service with invalid keytab URI scheme"); } catch (IllegalArgumentException e) { - assertEquals( - String.format(RestApiErrorMessages.ERROR_KEYTAB_URI_SCHEME_INVALID, - kp.getKeytab()), - e.getMessage()); + Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage()); } - kp.setKeytab("/ blank / in / paths"); - try { - ServiceApiUtil.validateKerberosPrincipal(app.getKerberosPrincipal()); - Assert.fail(EXCEPTION_PREFIX + "service with invalid keytab"); - } catch (IllegalArgumentException e) { - // strip out the %s at the end of the RestApiErrorMessages string constant - assertTrue(e.getMessage().contains( - RestApiErrorMessages.ERROR_KEYTAB_URI_INVALID.substring(0, - RestApiErrorMessages.ERROR_KEYTAB_URI_INVALID.length() - 2))); - } - - kp.setKeytab("file:///tmp/a.keytab"); - // now it should succeed + // Keytab with no URI scheme should succeed too + kp.setKeytab("/some/path"); try { ServiceApiUtil.validateKerberosPrincipal(app.getKerberosPrincipal()); } catch (IllegalArgumentException e) {