YARN-6655. Validate yarn native services application submission side to ensure that the hostname should be less than 63 characters. Contributed by Billie Rinaldi

This commit is contained in:
Jian He 2017-06-29 13:11:06 -07:00
parent e86d828548
commit 59a9b4f438
10 changed files with 155 additions and 37 deletions

View File

@ -153,7 +153,7 @@ definitions:
properties:
name:
type: string
description: A unique application name.
description: A unique application name. If Registry DNS is enabled, the max length is 63 characters.
id:
type: string
description: A unique application id.
@ -255,7 +255,7 @@ definitions:
properties:
name:
type: string
description: Name of the application component (mandatory).
description: Name of the application component (mandatory). If Registry DNS is enabled, the max length is 63 characters. If unique component support is enabled, the max length is lowered to 44 characters.
dependencies:
type: array
items:

View File

@ -637,7 +637,8 @@ private Application getApplicationFromArgs(String clusterName,
public int actionBuild(Application application) throws YarnException,
IOException {
Path appDir = checkAppNotExistOnHdfs(application);
ServiceApiUtil.validateAndResolveApplication(application, sliderFileSystem);
ServiceApiUtil.validateAndResolveApplication(application,
sliderFileSystem, getConfig());
persistApp(appDir, application);
deployedClusterName = application.getName();
return EXIT_SUCCESS;
@ -647,7 +648,8 @@ public ApplicationId actionCreate(Application application)
throws IOException, YarnException {
String appName = application.getName();
validateClusterName(appName);
ServiceApiUtil.validateAndResolveApplication(application, sliderFileSystem);
ServiceApiUtil.validateAndResolveApplication(application,
sliderFileSystem, getConfig());
verifyNoLiveApp(appName, "Create");
Path appDir = checkAppNotExistOnHdfs(application);
@ -1778,7 +1780,8 @@ public int actionStart(String appName, ActionThawArgs thaw)
Path appDir = checkAppExistOnHdfs(appName);
Application application = ServiceApiUtil.loadApplication(sliderFileSystem,
appName);
ServiceApiUtil.validateAndResolveApplication(application, sliderFileSystem);
ServiceApiUtil.validateAndResolveApplication(application,
sliderFileSystem, getConfig());
// see if it is actually running and bail out;
verifyNoLiveApp(appName, "Thaw");
ApplicationId appId = submitApp(application);

View File

@ -33,6 +33,7 @@
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.registry.client.api.RegistryConstants;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ExitUtil;

View File

@ -21,8 +21,13 @@ public interface RestApiErrorMessages {
String ERROR_APPLICATION_NAME_INVALID =
"Application name is either empty or not provided";
String ERROR_APPLICATION_NAME_INVALID_FORMAT =
"Application name %s is not valid - only lower case letters, digits,"
+ " underscore and hyphen are allowed";
"Application name %s is not valid - only lower case letters, digits, " +
"underscore and hyphen are allowed, and the name must be no more " +
"than 63 characters";
String ERROR_COMPONENT_NAME_INVALID =
"Component name must be no more than %s characters: %s";
String ERROR_USER_NAME_INVALID =
"User name must be no more than 63 characters";
String ERROR_APPLICATION_NOT_RUNNING = "Application not running";
String ERROR_APPLICATION_DOES_NOT_EXIST = "Application not found";

View File

@ -22,6 +22,8 @@
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.registry.client.api.RegistryConstants;
import org.apache.hadoop.registry.client.binding.RegistryUtils;
import org.apache.slider.api.resource.Application;
import org.apache.slider.api.resource.Artifact;
import org.apache.slider.api.resource.Component;
@ -60,12 +62,22 @@ public static void setJsonSerDeser(JsonSerDeser jsd) {
@VisibleForTesting
public static void validateAndResolveApplication(Application application,
SliderFileSystem fs) throws IOException {
SliderFileSystem fs, org.apache.hadoop.conf.Configuration conf) throws
IOException {
boolean dnsEnabled = conf.getBoolean(RegistryConstants.KEY_DNS_ENABLED,
RegistryConstants.DEFAULT_DNS_ENABLED);
if (dnsEnabled && RegistryUtils.currentUser().length() > RegistryConstants
.MAX_FQDN_LABEL_LENGTH) {
throw new IllegalArgumentException(RestApiErrorMessages
.ERROR_USER_NAME_INVALID);
}
if (StringUtils.isEmpty(application.getName())) {
throw new IllegalArgumentException(
RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID);
}
if (!SliderUtils.isClusternameValid(application.getName())) {
if (!SliderUtils.isClusternameValid(application.getName()) || (dnsEnabled
&& application.getName().length() > RegistryConstants
.MAX_FQDN_LABEL_LENGTH)) {
throw new IllegalArgumentException(String.format(
RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID_FORMAT,
application.getName()));
@ -108,6 +120,14 @@ public static void validateAndResolveApplication(Application application,
List<Component> componentsToRemove = new ArrayList<>();
List<Component> componentsToAdd = new ArrayList<>();
for (Component comp : application.getComponents()) {
int maxCompLength = RegistryConstants.MAX_FQDN_LABEL_LENGTH;
if (comp.getUniqueComponentSupport()) {
maxCompLength = maxCompLength - Long.toString(Long.MAX_VALUE).length();
}
if (dnsEnabled && comp.getName().length() > maxCompLength) {
throw new IllegalArgumentException(String.format(RestApiErrorMessages
.ERROR_COMPONENT_NAME_INVALID, maxCompLength, comp.getName()));
}
if (componentNames.contains(comp.getName())) {
throw new IllegalArgumentException("Component name collision: " +
comp.getName());

View File

@ -20,6 +20,7 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.slider.api.resource.Application;
import org.apache.slider.api.resource.ConfigFile;
import org.apache.slider.api.resource.ConfigFile.TypeEnum;
@ -95,7 +96,8 @@ public void testOverride() throws Throwable {
expect(sfs.buildClusterDirPath(anyObject())).andReturn(
new Path("cluster_dir_path")).anyTimes();
replay(sfs, mockFs);
ServiceApiUtil.validateAndResolveApplication(orig, sfs);
ServiceApiUtil.validateAndResolveApplication(orig, sfs, new
YarnConfiguration());
global = orig.getConfiguration();
LOG.info("global = {}", global);
@ -179,7 +181,8 @@ public void testOverrideExternalConfiguration() throws IOException {
new Path("cluster_dir_path")).anyTimes();
replay(sfs, mockFs);
Application ext = ExampleAppJson.loadResource(APP_JSON);
ServiceApiUtil.validateAndResolveApplication(ext, sfs);
ServiceApiUtil.validateAndResolveApplication(ext, sfs, new
YarnConfiguration());
reset(sfs, mockFs);
// perform the resolution on original application
@ -192,7 +195,8 @@ public void testOverrideExternalConfiguration() throws IOException {
.anyTimes();
replay(sfs, mockFs, jsonSerDeser);
ServiceApiUtil.setJsonSerDeser(jsonSerDeser);
ServiceApiUtil.validateAndResolveApplication(orig, sfs);
ServiceApiUtil.validateAndResolveApplication(orig, sfs, new
YarnConfiguration());
global = orig.getConfiguration();
assertEquals(0, global.getProperties().size());

View File

@ -20,6 +20,7 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.slider.api.resource.Application;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.util.ServiceApiUtil;
@ -71,7 +72,8 @@ public void testLoadResource() throws Throwable {
new Path("cluster_dir_path")).anyTimes();
replay(sfs, mockFs);
ServiceApiUtil.validateAndResolveApplication(application, sfs);
ServiceApiUtil.validateAndResolveApplication(application, sfs,
new YarnConfiguration());
} catch (Exception e) {
throw new Exception("exception loading " + resource + ":" + e.toString());
}

View File

@ -133,7 +133,7 @@ protected AppStateBindingInfo buildBindingInfo() throws IOException {
AppStateBindingInfo binding = new AppStateBindingInfo();
binding.application = buildApplication();
ServiceApiUtil.validateAndResolveApplication(binding.application,
sliderFileSystem);
sliderFileSystem, SliderUtils.createConfiguration());
//binding.roles = new ArrayList<>(factory.ROLES);
binding.fs = fs;
binding.historyPath = historyPath;

View File

@ -19,6 +19,8 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.registry.client.api.RegistryConstants;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.slider.api.resource.Application;
import org.apache.slider.api.resource.Artifact;
import org.apache.slider.api.resource.Component;
@ -29,6 +31,7 @@
import org.apache.slider.util.RestApiErrorMessages;
import org.apache.slider.util.ServiceApiUtil;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -61,26 +64,42 @@ public class TestServiceApiUtil {
private static final String NO_EXCEPTION_PREFIX = "Should not have thrown " +
"exception: ";
private static final String LEN_64_STR =
"abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz01";
private static final YarnConfiguration CONF_DEFAULT_DNS = new
YarnConfiguration();
private static final YarnConfiguration CONF_DNS_ENABLED = new
YarnConfiguration();
@BeforeClass
public static void init() {
CONF_DNS_ENABLED.setBoolean(RegistryConstants.KEY_DNS_ENABLED, true);
}
@Test(timeout = 90000)
public void testResourceValidation() throws Exception {
assertEquals(RegistryConstants.MAX_FQDN_LABEL_LENGTH + 1, LEN_64_STR
.length());
SliderFileSystem sfs = initMock(null);
Application app = new Application();
// no name
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "application with no name");
} catch (IllegalArgumentException e) {
assertEquals(ERROR_APPLICATION_NAME_INVALID, e.getMessage());
}
// bad format name
String[] badNames = {"4finance", "Finance", "finance@home"};
String[] badNames = {"4finance", "Finance", "finance@home", LEN_64_STR};
for (String badName : badNames) {
app.setName(badName);
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "application with bad name " + badName);
} catch (IllegalArgumentException e) {
assertEquals(String.format(
@ -89,9 +108,20 @@ public void testResourceValidation() throws Exception {
}
// launch command not specified
app.setName("finance_home");
app.setName(LEN_64_STR);
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DEFAULT_DNS);
Assert.fail(EXCEPTION_PREFIX + "application with no launch command");
} catch (IllegalArgumentException e) {
assertEquals(RestApiErrorMessages.ERROR_ABSENT_LAUNCH_COMMAND,
e.getMessage());
}
// launch command not specified
app.setName(LEN_64_STR.substring(0, RegistryConstants
.MAX_FQDN_LABEL_LENGTH));
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "application with no launch command");
} catch (IllegalArgumentException e) {
assertEquals(RestApiErrorMessages.ERROR_ABSENT_LAUNCH_COMMAND,
@ -101,7 +131,7 @@ public void testResourceValidation() throws Exception {
// resource not specified
app.setLaunchCommand("sleep 3600");
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "application with no resource");
} catch (IllegalArgumentException e) {
assertEquals(String.format(
@ -113,7 +143,7 @@ public void testResourceValidation() throws Exception {
Resource res = new Resource();
app.setResource(res);
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "application with no memory");
} catch (IllegalArgumentException e) {
assertEquals(String.format(
@ -125,7 +155,7 @@ public void testResourceValidation() throws Exception {
res.setMemory("100mb");
res.setCpus(-2);
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
Assert.fail(
EXCEPTION_PREFIX + "application with invalid no of cpus");
} catch (IllegalArgumentException e) {
@ -137,7 +167,7 @@ public void testResourceValidation() throws Exception {
// number of containers not specified
res.setCpus(2);
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "application with no container count");
} catch (IllegalArgumentException e) {
Assert.assertTrue(e.getMessage()
@ -147,7 +177,7 @@ public void testResourceValidation() throws Exception {
// specifying profile along with cpus/memory raises exception
res.setProfile("hbase_finance_large");
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX
+ "application with resource profile along with cpus/memory");
} catch (IllegalArgumentException e) {
@ -162,7 +192,7 @@ public void testResourceValidation() throws Exception {
res.setCpus(null);
res.setMemory(null);
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "application with resource profile only");
} catch (IllegalArgumentException e) {
assertEquals(ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET,
@ -176,7 +206,7 @@ public void testResourceValidation() throws Exception {
// null number of containers
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "null number of containers");
} catch (IllegalArgumentException e) {
Assert.assertTrue(e.getMessage()
@ -186,7 +216,7 @@ public void testResourceValidation() throws Exception {
// negative number of containers
app.setNumberOfContainers(-1L);
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "negative number of containers");
} catch (IllegalArgumentException e) {
Assert.assertTrue(e.getMessage()
@ -196,7 +226,7 @@ public void testResourceValidation() throws Exception {
// everything valid here
app.setNumberOfContainers(5L);
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
} catch (IllegalArgumentException e) {
LOG.error("application attributes specified should be valid here", e);
Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
@ -218,7 +248,7 @@ public void testArtifacts() throws IOException {
Artifact artifact = new Artifact();
app.setArtifact(artifact);
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "application with no artifact id");
} catch (IllegalArgumentException e) {
assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
@ -227,7 +257,7 @@ public void testArtifacts() throws IOException {
// no artifact id fails with APPLICATION type
artifact.setType(Artifact.TypeEnum.APPLICATION);
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "application with no artifact id");
} catch (IllegalArgumentException e) {
assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
@ -236,7 +266,7 @@ public void testArtifacts() throws IOException {
// no artifact id fails with TARBALL type
artifact.setType(Artifact.TypeEnum.TARBALL);
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "application with no artifact id");
} catch (IllegalArgumentException e) {
assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
@ -246,7 +276,7 @@ public void testArtifacts() throws IOException {
artifact.setType(Artifact.TypeEnum.DOCKER);
artifact.setId("docker.io/centos:centos7");
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
} catch (IllegalArgumentException e) {
LOG.error("application attributes specified should be valid here", e);
Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
@ -314,7 +344,7 @@ public void testExternalApplication() throws IOException {
app.setArtifact(artifact);
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
} catch (IllegalArgumentException e) {
Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
}
@ -333,7 +363,7 @@ public void testDuplicateComponents() throws IOException {
// duplicate component name fails
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "application with component collision");
} catch (IllegalArgumentException e) {
assertEquals("Component name collision: " + compName, e.getMessage());
@ -353,7 +383,7 @@ public void testExternalDuplicateComponent() throws IOException {
// duplicate component name okay in the case of APPLICATION component
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
} catch (IllegalArgumentException e) {
Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
}
@ -371,7 +401,7 @@ public void testExternalComponent() throws IOException {
app.setArtifact(artifact);
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
} catch (IllegalArgumentException e) {
Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
}
@ -384,7 +414,7 @@ public void testExternalComponent() throws IOException {
app.getComponent("comp2").setArtifact(artifact);
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs);
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
} catch (IllegalArgumentException e) {
Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
}
@ -441,4 +471,52 @@ public void testDependencySorting() throws IOException {
.getMessage());
}
}
@Test
public void testInvalidComponent() throws IOException {
SliderFileSystem sfs = initMock(null);
testComponent(sfs, false);
testComponent(sfs, true);
}
private static void testComponent(SliderFileSystem sfs, boolean unique)
throws IOException {
int maxLen = RegistryConstants.MAX_FQDN_LABEL_LENGTH;
if (unique) {
assertEquals(19, Long.toString(Long.MAX_VALUE).length());
maxLen = maxLen - Long.toString(Long.MAX_VALUE).length();
}
String compName = LEN_64_STR.substring(0, maxLen + 1);
Application app = createValidApplication(null);
app.addComponent(createValidComponent(compName).uniqueComponentSupport(
unique));
// invalid component name fails if dns is enabled
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
Assert.fail(EXCEPTION_PREFIX + "application with invalid component name");
} catch (IllegalArgumentException e) {
assertEquals(String.format(RestApiErrorMessages
.ERROR_COMPONENT_NAME_INVALID, maxLen, compName), e.getMessage());
}
// does not fail if dns is disabled
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DEFAULT_DNS);
} catch (IllegalArgumentException e) {
Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
}
compName = LEN_64_STR.substring(0, maxLen);
app = createValidApplication(null);
app.addComponent(createValidComponent(compName).uniqueComponentSupport(
unique));
// does not fail
try {
ServiceApiUtil.validateAndResolveApplication(app, sfs, CONF_DNS_ENABLED);
} catch (IllegalArgumentException e) {
Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
}
}
}

View File

@ -77,6 +77,11 @@ public interface RegistryConstants {
*/
String KEY_DNS_DOMAIN = DNS_PREFIX + "domain-name";
/**
* Max length of a label (node delimited by a dot in the FQDN).
*/
int MAX_FQDN_LABEL_LENGTH = 63;
/**
* DNS bind address.
*/