From 0c559b27820d12ebe5c315c6e2d1685c6de6bd33 Mon Sep 17 00:00:00 2001 From: Billie Rinaldi Date: Wed, 24 Jan 2018 13:50:47 -0800 Subject: [PATCH] YARN-7777. Fix user name format in YARN Registry DNS name. Contributed by Jian He --- .../hadoop/yarn/service/ServiceScheduler.java | 11 +++------- .../yarn/service/utils/ServiceApiUtil.java | 15 +++++++++---- .../client/binding/RegistryPathUtils.java | 20 +++++++++++++++++ .../client/binding/RegistryUtils.java | 5 ++++- .../dns/BaseServiceRecordProcessor.java | 22 ++----------------- .../client/binding/TestRegistryPathUtils.java | 10 ++++++++- .../markdown/yarn-service/Configurations.md | 2 +- 7 files changed, 50 insertions(+), 35 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java index 6cf4e1413c..dfe9808a02 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java @@ -392,14 +392,9 @@ private void initGlobalTokensForSubstitute(ServiceContext context) { // ZK globalTokens.put(ServiceApiConstants.CLUSTER_ZK_QUORUM, getConfig() .getTrimmed(KEY_REGISTRY_ZK_QUORUM, DEFAULT_REGISTRY_ZK_QUORUM)); - String user = null; - try { - user = UserGroupInformation.getCurrentUser().getShortUserName(); - } catch (IOException e) { - LOG.error("Failed to get user.", e); - } - globalTokens - .put(SERVICE_ZK_PATH, ServiceRegistryUtils.mkServiceHomePath(user, app.getName())); + String user = RegistryUtils.currentUser(); + globalTokens.put(SERVICE_ZK_PATH, + ServiceRegistryUtils.mkServiceHomePath(user, app.getName())); globalTokens.put(ServiceApiConstants.USER, user); String dnsDomain = getConfig().getTrimmed(KEY_DNS_DOMAIN); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java index 7f85c95176..a5716bd3a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java @@ -61,6 +61,9 @@ public class ServiceApiUtil { private static final PatternValidator namePattern = new PatternValidator("[a-z][a-z0-9-]*"); + private static final PatternValidator userNamePattern + = new PatternValidator("[a-z][a-z0-9-.]*"); + @VisibleForTesting public static void setJsonSerDeser(JsonSerDeser jsd) { jsonSerDeser = jsd; @@ -72,11 +75,15 @@ public static void validateAndResolveService(Service service, IOException { boolean dnsEnabled = conf.getBoolean(RegistryConstants.KEY_DNS_ENABLED, RegistryConstants.DEFAULT_DNS_ENABLED); - if (dnsEnabled && RegistryUtils.currentUser().length() > RegistryConstants - .MAX_FQDN_LABEL_LENGTH) { - throw new IllegalArgumentException(RestApiErrorMessages - .ERROR_USER_NAME_INVALID); + if (dnsEnabled) { + if (RegistryUtils.currentUser().length() + > RegistryConstants.MAX_FQDN_LABEL_LENGTH) { + throw new IllegalArgumentException( + RestApiErrorMessages.ERROR_USER_NAME_INVALID); + } + userNamePattern.validate(RegistryUtils.currentUser()); } + if (StringUtils.isEmpty(service.getName())) { throw new IllegalArgumentException( RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java index 5fa45f913d..b8e9ba1bd7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java @@ -24,11 +24,13 @@ import org.apache.hadoop.fs.PathNotFoundException; import org.apache.hadoop.registry.client.exceptions.InvalidPathnameException; import org.apache.hadoop.registry.client.impl.zk.RegistryInternalConstants; +import org.apache.hadoop.registry.server.dns.BaseServiceRecordProcessor; import org.apache.zookeeper.common.PathUtils; import java.net.IDN; import java.util.ArrayList; import java.util.List; +import java.util.regex.Matcher; import java.util.regex.Pattern; /** @@ -45,6 +47,9 @@ public class RegistryPathUtils { private static final Pattern PATH_ENTRY_VALIDATION_PATTERN = Pattern.compile(RegistryInternalConstants.VALID_PATH_ENTRY_PATTERN); + private static final Pattern USER_NAME = + Pattern.compile("/users/([a-z][a-z0-9-.]*)"); + /** * Validate ZK path with the path itself included in * the exception text @@ -215,4 +220,19 @@ public static String encodeForRegistry(String element) { public static String encodeYarnID(String yarnId) { return yarnId.replace("container", "ctr").replace("_", "-"); } + + /** + * Return the username found in the ZK path. + * + * @param recPath the ZK recPath. + * @return the user name. + */ + public static String getUsername(String recPath) { + String user = "anonymous"; + Matcher matcher = USER_NAME.matcher(recPath); + if (matcher.find()) { + user = matcher.group(1); + } + return user; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryUtils.java index 858b6b1235..4ef7b8d404 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryUtils.java @@ -296,7 +296,10 @@ public static String getCurrentUsernameUnencoded(String env_hadoop_username) { */ public static String currentUser() { String shortUserName = currentUsernameUnencoded(); - return encodeForRegistry(shortUserName); + String encodedName = encodeForRegistry(shortUserName); + // DNS name doesn't allow "_", replace it with "-" + encodedName = RegistryUtils.convertUsername(encodedName); + return encodedName.replace("_", "-"); } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java index fd5c74f649..51ae99a55e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java @@ -36,8 +36,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; /** * Provides common service record processing logic. @@ -51,7 +49,6 @@ public abstract class BaseServiceRecordProcessor private String path; private String domain; - private static final Pattern USER_NAME = Pattern.compile("/users/(\\w*)/?"); private static final String YARN_SERVICE_API_PREFIX = "classpath:org.apache.hadoop.yarn.service."; private static final String HTTP_API_TYPE = "http://"; @@ -75,21 +72,6 @@ public BaseServiceRecordProcessor(ServiceRecord record, String path, initTypeToInfoMapping(record); } - /** - * Return the username found in the ZK path. - * - * @param recPath the ZK recPath. - * @return the user name. - */ - protected String getUsername(String recPath) { - String user = "anonymous"; - Matcher matcher = USER_NAME.matcher(recPath); - if (matcher.find()) { - user = matcher.group(1); - } - return user; - } - /** * Return the IPv6 mapped address for the provided IPv4 address. Utilized * to create corresponding AAAA records. @@ -300,7 +282,7 @@ protected Name getContainerName() String service = RegistryPathUtils.lastPathEntry( RegistryPathUtils.parentOf(RegistryPathUtils.parentOf(getPath()))); String description = getRecord().description.toLowerCase(); - String user = getUsername(getPath()); + String user = RegistryPathUtils.getUsername(getPath()); return Name.fromString(MessageFormat.format("{0}.{1}.{2}.{3}", description, service, @@ -352,7 +334,7 @@ public ApplicationRecordDescriptor(ServiceRecord record, * @throws TextParseException */ protected Name getServiceName() throws TextParseException { - String user = getUsername(getPath()); + String user = RegistryPathUtils.getUsername(getPath()); String service = String.format("%s.%s.%s", RegistryPathUtils.lastPathEntry(getPath()), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryPathUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryPathUtils.java index 9a24f1c9c8..4346c9a787 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryPathUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryPathUtils.java @@ -80,7 +80,15 @@ public void testPaths() throws Throwable { assertCreatedPathEquals("/alice", "/alice", "/"); } - + @Test + public void testGetUserFromPath() throws Exception { + assertEquals("bob", RegistryPathUtils + .getUsername("/registry/users/bob/services/yarn-service/test1/")); + assertEquals("bob-dev", RegistryPathUtils + .getUsername("/registry/users/bob-dev/services/yarn-service/test1")); + assertEquals("bob.dev", RegistryPathUtils + .getUsername("/registry/users/bob.dev/services/yarn-service/test1")); + } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md index a6fd998b8c..c2e6d260f0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md @@ -155,7 +155,7 @@ where `regionserver-0` is the actual component instance name assigned by the sys | Name | Description | | ------------ | ------------- | | SERVICE_NAME | name of the service defined by the user -| USER | user who submits the service | +| USER | user who submits the service. Note that user name which has "\_" will be converted to use "-", to conform with DNS hostname RFC format which doesn't allow "\_", and all characters will be lowercased E.g. "Bob_dev" will be converted to "bob-dev" | | DOMAIN | the domain name for the cluster | | COMPONENT_NAME | the name for a given component | | COMPONENT_INSTANCE_NAME | the name for a given component instance (i.e. container) |