big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-7777. Fix user name format in YARN Registry DNS name. Contributed by Jian He

Browse Source
This commit is contained in:
Billie Rinaldi 2018-01-24 13:50:47 -08:00
parent fa8cf4d1b4
commit 0c559b2782
7 changed files with 50 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
View File

@ -392,14 +392,9 @@ private void initGlobalTokensForSubstitute(ServiceContext context) {
// ZK
globalTokens.put(ServiceApiConstants.CLUSTER_ZK_QUORUM, getConfig()
.getTrimmed(KEY_REGISTRY_ZK_QUORUM, DEFAULT_REGISTRY_ZK_QUORUM));
String user = null;
try {
user = UserGroupInformation.getCurrentUser().getShortUserName();
} catch (IOException e) {
LOG.error("Failed to get user.", e);
}
globalTokens
.put(SERVICE_ZK_PATH, ServiceRegistryUtils.mkServiceHomePath(user, app.getName()));
String user = RegistryUtils.currentUser();
globalTokens.put(SERVICE_ZK_PATH,
ServiceRegistryUtils.mkServiceHomePath(user, app.getName()));
globalTokens.put(ServiceApiConstants.USER, user);
String dnsDomain = getConfig().getTrimmed(KEY_DNS_DOMAIN);

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
View File

@ -61,6 +61,9 @@ public class ServiceApiUtil {
private static final PatternValidator namePattern
= new PatternValidator("[a-z][a-z0-9-]*");
private static final PatternValidator userNamePattern
= new PatternValidator("[a-z][a-z0-9-.]*");
@VisibleForTesting
public static void setJsonSerDeser(JsonSerDeser jsd) {
jsonSerDeser = jsd;
@ -72,11 +75,15 @@ public static void validateAndResolveService(Service service,
IOException {
boolean dnsEnabled = conf.getBoolean(RegistryConstants.KEY_DNS_ENABLED,
RegistryConstants.DEFAULT_DNS_ENABLED);
if (dnsEnabled && RegistryUtils.currentUser().length() > RegistryConstants
.MAX_FQDN_LABEL_LENGTH) {
throw new IllegalArgumentException(RestApiErrorMessages
.ERROR_USER_NAME_INVALID);
if (dnsEnabled) {
if (RegistryUtils.currentUser().length()
> RegistryConstants.MAX_FQDN_LABEL_LENGTH) {
throw new IllegalArgumentException(
RestApiErrorMessages.ERROR_USER_NAME_INVALID);
}
userNamePattern.validate(RegistryUtils.currentUser());
}
if (StringUtils.isEmpty(service.getName())) {
throw new IllegalArgumentException(
RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID);

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java
View File

@ -24,11 +24,13 @@
import org.apache.hadoop.fs.PathNotFoundException;
import org.apache.hadoop.registry.client.exceptions.InvalidPathnameException;
import org.apache.hadoop.registry.client.impl.zk.RegistryInternalConstants;
import org.apache.hadoop.registry.server.dns.BaseServiceRecordProcessor;
import org.apache.zookeeper.common.PathUtils;
import java.net.IDN;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
@ -45,6 +47,9 @@ public class RegistryPathUtils {
private static final Pattern PATH_ENTRY_VALIDATION_PATTERN =
Pattern.compile(RegistryInternalConstants.VALID_PATH_ENTRY_PATTERN);
private static final Pattern USER_NAME =
Pattern.compile("/users/([a-z][a-z0-9-.]*)");
/**
* Validate ZK path with the path itself included in
* the exception text
@ -215,4 +220,19 @@ public static String encodeForRegistry(String element) {
public static String encodeYarnID(String yarnId) {
return yarnId.replace("container", "ctr").replace("_", "-");
}
/**
* Return the username found in the ZK path.
*
* @param recPath the ZK recPath.
* @return the user name.
*/
public static String getUsername(String recPath) {
String user = "anonymous";
Matcher matcher = USER_NAME.matcher(recPath);
if (matcher.find()) {
user = matcher.group(1);
}
return user;
}
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryUtils.java
View File

@ -296,7 +296,10 @@ public static String getCurrentUsernameUnencoded(String env_hadoop_username) {
*/
public static String currentUser() {
String shortUserName = currentUsernameUnencoded();
return encodeForRegistry(shortUserName);
String encodedName = encodeForRegistry(shortUserName);
// DNS name doesn't allow "_", replace it with "-"
encodedName = RegistryUtils.convertUsername(encodedName);
return encodedName.replace("_", "-");
}
/**

22
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java
View File

@ -36,8 +36,6 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Provides common service record processing logic.
@ -51,7 +49,6 @@ public abstract class BaseServiceRecordProcessor
private String path;
private String domain;
private static final Pattern USER_NAME = Pattern.compile("/users/(\\w*)/?");
private static final String YARN_SERVICE_API_PREFIX =
"classpath:org.apache.hadoop.yarn.service.";
private static final String HTTP_API_TYPE = "http://";
@ -75,21 +72,6 @@ public BaseServiceRecordProcessor(ServiceRecord record, String path,
initTypeToInfoMapping(record);
}
/**
* Return the username found in the ZK path.
*
* @param recPath the ZK recPath.
* @return the user name.
*/
protected String getUsername(String recPath) {
String user = "anonymous";
Matcher matcher = USER_NAME.matcher(recPath);
if (matcher.find()) {
user = matcher.group(1);
}
return user;
}
/**
* Return the IPv6 mapped address for the provided IPv4 address. Utilized
* to create corresponding AAAA records.
@ -300,7 +282,7 @@ protected Name getContainerName()
String service = RegistryPathUtils.lastPathEntry(
RegistryPathUtils.parentOf(RegistryPathUtils.parentOf(getPath())));
String description = getRecord().description.toLowerCase();
String user = getUsername(getPath());
String user = RegistryPathUtils.getUsername(getPath());
return Name.fromString(MessageFormat.format("{0}.{1}.{2}.{3}",
description,
service,
@ -352,7 +334,7 @@ public ApplicationRecordDescriptor(ServiceRecord record,
* @throws TextParseException
*/
protected Name getServiceName() throws TextParseException {
String user = getUsername(getPath());
String user = RegistryPathUtils.getUsername(getPath());
String service =
String.format("%s.%s.%s",
RegistryPathUtils.lastPathEntry(getPath()),

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryPathUtils.java
View File

@ -80,7 +80,15 @@ public void testPaths() throws Throwable {
assertCreatedPathEquals("/alice", "/alice", "/");
}
@Test
public void testGetUserFromPath() throws Exception {
assertEquals("bob", RegistryPathUtils
.getUsername("/registry/users/bob/services/yarn-service/test1/"));
assertEquals("bob-dev", RegistryPathUtils
.getUsername("/registry/users/bob-dev/services/yarn-service/test1"));
assertEquals("bob.dev", RegistryPathUtils
.getUsername("/registry/users/bob.dev/services/yarn-service/test1"));
}
@Test

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
View File

@ -155,7 +155,7 @@ where `regionserver-0` is the actual component instance name assigned by the sys
| Name | Description |
| ------------ | ------------- |
| SERVICE_NAME | name of the service defined by the user
| USER | user who submits the service |
| USER | user who submits the service. Note that user name which has "\_" will be converted to use "-", to conform with DNS hostname RFC format which doesn't allow "\_", and all characters will be lowercased E.g. "Bob_dev" will be converted to "bob-dev" |
| DOMAIN | the domain name for the cluster |
| COMPONENT_NAME | the name for a given component |
| COMPONENT_INSTANCE_NAME | the name for a given component instance (i.e. container) |