big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert "YARN-6804. [YARN core changes] Allow custom hostname for docker containers in native services. Contributed by Billie Rinaldi"

Browse Source 
This reverts commit 4a771d9010.
This commit is contained in:
Brahma Reddy Battula 2017-07-23 12:56:18 +08:00
parent 465c213439
commit 2054324d47
9 changed files with 44 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java
View File

@ -213,6 +213,6 @@ public static String encodeForRegistry(String element) {
* @return a string suitable for use in registry paths. * @return a string suitable for use in registry paths.
*/ */
public static String encodeYarnID(String yarnId) { public static String encodeYarnID(String yarnId) {
return yarnId.replace("container", "ctr").replace("_", "-"); return yarnId.replace("_", "-");
} }
} }

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/Endpoint.java
View File

@ -19,7 +19,7 @@
package org.apache.hadoop.registry.client.types; package org.apache.hadoop.registry.client.types;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.InterfaceStability;
@ -46,7 +46,7 @@
@InterfaceAudience.Public @InterfaceAudience.Public
@InterfaceStability.Evolving @InterfaceStability.Evolving
@JsonIgnoreProperties(ignoreUnknown = true) @JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(JsonInclude.Include.NON_NULL) @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public final class Endpoint implements Cloneable { public final class Endpoint implements Cloneable {
/** /**

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/ServiceRecord.java
View File

@ -20,7 +20,7 @@
import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnyGetter;
import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonAnySetter;
import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.InterfaceStability;
@ -37,7 +37,7 @@
*/ */
@InterfaceAudience.Public @InterfaceAudience.Public
@InterfaceStability.Evolving @InterfaceStability.Evolving
@JsonInclude(JsonInclude.Include.NON_NULL) @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class ServiceRecord implements Cloneable { public class ServiceRecord implements Cloneable {
/** /**

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
View File

@ -51,10 +51,6 @@
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId> <artifactId>hadoop-yarn-api</artifactId>
</dependency> </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-registry</artifactId>
</dependency>
<dependency> <dependency>
<groupId>javax.xml.bind</groupId> <groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId> <artifactId>jaxb-api</artifactId>

67
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
View File

@ -27,7 +27,6 @@
import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.StringUtils;
@ -102,11 +101,6 @@
* property. * property.
* </li> * </li>
* <li> * <li>
* {@code YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_HOSTNAME} sets the
* hostname to be used by the Docker container. If not specified, a
* hostname will be derived from the container ID.
* </li>
* <li>
* {@code YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER} * {@code YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER}
* controls whether the Docker container is a privileged container. In order * controls whether the Docker container is a privileged container. In order
* to use privileged containers, the * to use privileged containers, the
@ -140,10 +134,6 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
"^(([a-zA-Z0-9.-]+)(:\\d+)?/)?([a-z0-9_./-]+)(:[\\w.-]+)?$"; "^(([a-zA-Z0-9.-]+)(:\\d+)?/)?([a-z0-9_./-]+)(:[\\w.-]+)?$";
private static final Pattern dockerImagePattern = private static final Pattern dockerImagePattern =
Pattern.compile(DOCKER_IMAGE_PATTERN); Pattern.compile(DOCKER_IMAGE_PATTERN);
public static final String HOSTNAME_PATTERN =
"^[a-zA-Z0-9][a-zA-Z0-9_.-]+$";
private static final Pattern hostnamePattern = Pattern.compile(
HOSTNAME_PATTERN);
@InterfaceAudience.Private @InterfaceAudience.Private
public static final String ENV_DOCKER_CONTAINER_IMAGE = public static final String ENV_DOCKER_CONTAINER_IMAGE =
@ -157,10 +147,6 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
@InterfaceAudience.Private @InterfaceAudience.Private
public static final String ENV_DOCKER_CONTAINER_NETWORK = public static final String ENV_DOCKER_CONTAINER_NETWORK =
"YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK"; "YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK";
@InterfaceAudience.Private
public static final String ENV_DOCKER_CONTAINER_HOSTNAME =
"YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_HOSTNAME";
@InterfaceAudience.Private
public static final String ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER = public static final String ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER =
"YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER"; "YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER";
@InterfaceAudience.Private @InterfaceAudience.Private
@ -225,7 +211,9 @@ public DockerLinuxContainerRuntime(PrivilegedOperationExecutor
this.privilegedOperationExecutor = privilegedOperationExecutor; this.privilegedOperationExecutor = privilegedOperationExecutor;
if (cGroupsHandler == null) { if (cGroupsHandler == null) {
LOG.info("cGroupsHandler is null - cgroups not in use."); if (LOG.isInfoEnabled()) {
LOG.info("cGroupsHandler is null - cgroups not in use.");
}
} else { } else {
this.cGroupsHandler = cGroupsHandler; this.cGroupsHandler = cGroupsHandler;
} }
@ -279,29 +267,6 @@ private void validateContainerNetworkType(String network)
throw new ContainerExecutionException(msg); throw new ContainerExecutionException(msg);
} }
public static void validateHostname(String hostname) throws
ContainerExecutionException {
if (hostname != null && !hostname.isEmpty()) {
if (!hostnamePattern.matcher(hostname).matches()) {
throw new ContainerExecutionException("Hostname '" + hostname
+ "' doesn't match docker hostname pattern");
}
}
}
/** Set a DNS friendly hostname. */
private void setHostname(DockerRunCommand runCommand, String
containerIdStr, String name)
throws ContainerExecutionException {
if (name == null || name.isEmpty()) {
name = RegistryPathUtils.encodeYarnID(containerIdStr);
validateHostname(name);
}
LOG.info("setting hostname in container to: " + name);
runCommand.setHostname(name);
}
/** /**
* If CGROUPS in enabled and not set to none, then set the CGROUP parent for * If CGROUPS in enabled and not set to none, then set the CGROUP parent for
* the command instance. * the command instance.
@ -378,8 +343,10 @@ private boolean allowPrivilegedContainerExecution(Container container)
return false; return false;
} }
LOG.info("Privileged container requested for : " + container if (LOG.isInfoEnabled()) {
.getContainerId().toString()); LOG.info("Privileged container requested for : " + container
.getContainerId().toString());
}
//Ok, so we have been asked to run a privileged container. Security //Ok, so we have been asked to run a privileged container. Security
// checks need to be run. Each violation is an error. // checks need to be run. Each violation is an error.
@ -408,8 +375,10 @@ private boolean allowPrivilegedContainerExecution(Container container)
throw new ContainerExecutionException(message); throw new ContainerExecutionException(message);
} }
LOG.info("All checks pass. Launching privileged container for : " if (LOG.isInfoEnabled()) {
+ container.getContainerId().toString()); LOG.info("All checks pass. Launching privileged container for : "
+ container.getContainerId().toString());
}
return true; return true;
} }
@ -444,7 +413,6 @@ public void launchContainer(ContainerRuntimeContext ctx)
.getEnvironment(); .getEnvironment();
String imageName = environment.get(ENV_DOCKER_CONTAINER_IMAGE); String imageName = environment.get(ENV_DOCKER_CONTAINER_IMAGE);
String network = environment.get(ENV_DOCKER_CONTAINER_NETWORK); String network = environment.get(ENV_DOCKER_CONTAINER_NETWORK);
String hostname = environment.get(ENV_DOCKER_CONTAINER_HOSTNAME);
if(network == null || network.isEmpty()) { if(network == null || network.isEmpty()) {
network = defaultNetwork; network = defaultNetwork;
@ -452,8 +420,6 @@ public void launchContainer(ContainerRuntimeContext ctx)
validateContainerNetworkType(network); validateContainerNetworkType(network);
validateHostname(hostname);
validateImageName(imageName); validateImageName(imageName);
String containerIdStr = container.getContainerId().toString(); String containerIdStr = container.getContainerId().toString();
@ -484,13 +450,12 @@ public void launchContainer(ContainerRuntimeContext ctx)
runAsUser, imageName) runAsUser, imageName)
.detachOnRun() .detachOnRun()
.setContainerWorkDir(containerWorkDir.toString()) .setContainerWorkDir(containerWorkDir.toString())
.setNetworkType(network); .setNetworkType(network)
setHostname(runCommand, containerIdStr, hostname); .setCapabilities(capabilities)
runCommand.setCapabilities(capabilities)
.addMountLocation(CGROUPS_ROOT_DIRECTORY, .addMountLocation(CGROUPS_ROOT_DIRECTORY,
CGROUPS_ROOT_DIRECTORY + ":ro", false); CGROUPS_ROOT_DIRECTORY + ":ro", false);
List<String> allDirs = new ArrayList<>(containerLocalDirs); List<String> allDirs = new ArrayList<>(containerLocalDirs);
allDirs.addAll(filecacheDirs); allDirs.addAll(filecacheDirs);
allDirs.add(containerWorkDir.toString()); allDirs.add(containerWorkDir.toString());
allDirs.addAll(containerLogDirs); allDirs.addAll(containerLogDirs);
@ -528,7 +493,9 @@ public void launchContainer(ContainerRuntimeContext ctx)
ENV_DOCKER_CONTAINER_RUN_OVERRIDE_DISABLE); ENV_DOCKER_CONTAINER_RUN_OVERRIDE_DISABLE);
if (disableOverride != null && disableOverride.equals("true")) { if (disableOverride != null && disableOverride.equals("true")) {
LOG.info("command override disabled"); if (LOG.isInfoEnabled()) {
LOG.info("command override disabled");
}
} else { } else {
List<String> overrideCommands = new ArrayList<>(); List<String> overrideCommands = new ArrayList<>();
Path launchDst = Path launchDst =

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
View File

@ -91,12 +91,6 @@ public DockerRunCommand setCapabilities(Set<String> capabilties) {
return this; return this;
} }
public DockerRunCommand setHostname(String hostname) {
super.addCommandArguments("--hostname=" + hostname);
return this;
}
public DockerRunCommand addDevice(String sourceDevice, String public DockerRunCommand addDevice(String sourceDevice, String
destinationDevice) { destinationDevice) {
super.addCommandArguments("--device=" + sourceDevice + ":" + super.addCommandArguments("--device=" + sourceDevice + ":" +

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
View File

@ -1215,7 +1215,6 @@ char* sanitize_docker_command(const char *line) {
{"rm", no_argument, 0, 'r' }, {"rm", no_argument, 0, 'r' },
{"workdir", required_argument, 0, 'w' }, {"workdir", required_argument, 0, 'w' },
{"net", required_argument, 0, 'e' }, {"net", required_argument, 0, 'e' },
{"hostname", required_argument, 0, 'h' },
{"cgroup-parent", required_argument, 0, 'g' }, {"cgroup-parent", required_argument, 0, 'g' },
{"privileged", no_argument, 0, 'p' }, {"privileged", no_argument, 0, 'p' },
{"cap-add", required_argument, 0, 'a' }, {"cap-add", required_argument, 0, 'a' },
@ -1257,9 +1256,6 @@ char* sanitize_docker_command(const char *line) {
case 'e': case 'e':
quote_and_append_arg(&output, &output_size, "--net=", optarg); quote_and_append_arg(&output, &output_size, "--net=", optarg);
break; break;
case 'h':
quote_and_append_arg(&output, &output_size, "--hostname=", optarg);
break;
case 'v': case 'v':
quote_and_append_arg(&output, &output_size, "-v ", optarg); quote_and_append_arg(&output, &output_size, "-v ", optarg);
break; break;

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
View File

@ -1088,17 +1088,17 @@ void test_trim_function() {
void test_sanitize_docker_command() { void test_sanitize_docker_command() {
char *input[] = { char *input[] = {
"run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --hostname=test.host.name --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh", "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
"run --name=$CID --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --hostname=test.host.name --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh", "run --name=$CID --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
"run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --hostname=test.host.name --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu || touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh", "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu || touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
"run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --hostname=test.host.name --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu' || touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh", "run --name=cname --user=nobody -d --workdir=/yarn/local/cdir --privileged --rm --device=/sys/fs/cgroup/device:/sys/fs/cgroup/device --detach=true --cgroup-parent=/sys/fs/cgroup/cpu/yarn/cid --net=host --cap-drop=ALL --cap-add=SYS_CHROOT --cap-add=MKNOD --cap-add=SETFCAP --cap-add=SETPCAP --cap-add=FSETID --cap-add=CHOWN --cap-add=AUDIT_WRITE --cap-add=SETGID --cap-add=NET_RAW --cap-add=FOWNER --cap-add=SETUID --cap-add=DAC_OVERRIDE --cap-add=KILL --cap-add=NET_BIND_SERVICE -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /yarn/local/cdir:/yarn/local/cdir -v /yarn/local/usercache/test/:/yarn/local/usercache/test/ ubuntu' || touch /tmp/file # bash /yarn/local/usercache/test/appcache/aid/cid/launch_container.sh",
"run ''''''''" "run ''''''''"
}; };
char *expected_output[] = { char *expected_output[] = {
"run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --hostname='test.host.name' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ", "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
"run --name='$CID' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --hostname='test.host.name' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ", "run --name='$CID' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
"run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --hostname='test.host.name' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' '||' 'touch' '/tmp/file' '#' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ", "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu' '||' 'touch' '/tmp/file' '#' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
"run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --hostname='test.host.name' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu'\"'\"'' '||' 'touch' '/tmp/file' '#' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ", "run --name='cname' --user='nobody' -d --workdir='/yarn/local/cdir' --privileged --rm --device='/sys/fs/cgroup/device:/sys/fs/cgroup/device' --detach='true' --cgroup-parent='/sys/fs/cgroup/cpu/yarn/cid' --net='host' --cap-drop='ALL' --cap-add='SYS_CHROOT' --cap-add='MKNOD' --cap-add='SETFCAP' --cap-add='SETPCAP' --cap-add='FSETID' --cap-add='CHOWN' --cap-add='AUDIT_WRITE' --cap-add='SETGID' --cap-add='NET_RAW' --cap-add='FOWNER' --cap-add='SETUID' --cap-add='DAC_OVERRIDE' --cap-add='KILL' --cap-add='NET_BIND_SERVICE' -v '/sys/fs/cgroup:/sys/fs/cgroup:ro' -v '/yarn/local/cdir:/yarn/local/cdir' -v '/yarn/local/usercache/test/:/yarn/local/usercache/test/' 'ubuntu'\"'\"'' '||' 'touch' '/tmp/file' '#' 'bash' '/yarn/local/usercache/test/appcache/aid/cid/launch_container.sh' ",
"run ''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"'' ", "run ''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"''\"'\"'' ",
}; };

58
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
View File

@ -25,7 +25,6 @@
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
@ -70,7 +69,6 @@ public class TestDockerContainerRuntime {
private PrivilegedOperationExecutor mockExecutor; private PrivilegedOperationExecutor mockExecutor;
private CGroupsHandler mockCGroupsHandler; private CGroupsHandler mockCGroupsHandler;
private String containerId; private String containerId;
private String defaultHostname;
private Container container; private Container container;
private ContainerId cId; private ContainerId cId;
private ContainerLaunchContext context; private ContainerLaunchContext context;
@ -110,7 +108,6 @@ public void setup() {
.mock(PrivilegedOperationExecutor.class); .mock(PrivilegedOperationExecutor.class);
mockCGroupsHandler = Mockito.mock(CGroupsHandler.class); mockCGroupsHandler = Mockito.mock(CGroupsHandler.class);
containerId = "container_id"; containerId = "container_id";
defaultHostname = RegistryPathUtils.encodeYarnID(containerId);
container = mock(Container.class); container = mock(Container.class);
cId = mock(ContainerId.class); cId = mock(ContainerId.class);
context = mock(ContainerLaunchContext.class); context = mock(ContainerLaunchContext.class);
@ -290,7 +287,6 @@ public void testDockerContainerLaunch()
.append("--user=%2$s -d ") .append("--user=%2$s -d ")
.append("--workdir=%3$s ") .append("--workdir=%3$s ")
.append("--net=host ") .append("--net=host ")
.append("--hostname=" + defaultHostname + " ")
.append(getExpectedTestCapabilitiesArgumentString()) .append(getExpectedTestCapabilitiesArgumentString())
.append(getExpectedCGroupsMountString()) .append(getExpectedCGroupsMountString())
.append("-v %4$s:%4$s ") .append("-v %4$s:%4$s ")
@ -369,7 +365,7 @@ public void testContainerLaunchWithNetworkingDefaults()
String disallowedNetwork = "sdn" + Integer.toString(randEngine.nextInt()); String disallowedNetwork = "sdn" + Integer.toString(randEngine.nextInt());
try { try {
env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_NETWORK, env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK",
disallowedNetwork); disallowedNetwork);
runtime.launchContainer(builder.build()); runtime.launchContainer(builder.build());
Assert.fail("Network was expected to be disallowed: " + Assert.fail("Network was expected to be disallowed: " +
@ -382,11 +378,8 @@ public void testContainerLaunchWithNetworkingDefaults()
.DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS.length; .DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS.length;
String allowedNetwork = YarnConfiguration String allowedNetwork = YarnConfiguration
.DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS[randEngine.nextInt(size)]; .DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS[randEngine.nextInt(size)];
env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_NETWORK, env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK",
allowedNetwork); allowedNetwork);
String expectedHostname = "test.hostname";
env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_HOSTNAME,
expectedHostname);
//this should cause no failures. //this should cause no failures.
@ -400,7 +393,6 @@ public void testContainerLaunchWithNetworkingDefaults()
new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ") new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ")
.append("--workdir=%3$s ") .append("--workdir=%3$s ")
.append("--net=" + allowedNetwork + " ") .append("--net=" + allowedNetwork + " ")
.append("--hostname=" + expectedHostname + " ")
.append(getExpectedTestCapabilitiesArgumentString()) .append(getExpectedTestCapabilitiesArgumentString())
.append(getExpectedCGroupsMountString()) .append(getExpectedCGroupsMountString())
.append("-v %4$s:%4$s ").append("-v %5$s:%5$s ") .append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
@ -456,7 +448,6 @@ public void testContainerLaunchWithCustomNetworks()
new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ") new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ")
.append("--workdir=%3$s ") .append("--workdir=%3$s ")
.append("--net=" + customNetwork1 + " ") .append("--net=" + customNetwork1 + " ")
.append("--hostname=" + defaultHostname + " ")
.append(getExpectedTestCapabilitiesArgumentString()) .append(getExpectedTestCapabilitiesArgumentString())
.append(getExpectedCGroupsMountString()) .append(getExpectedCGroupsMountString())
.append("-v %4$s:%4$s ").append("-v %5$s:%5$s ") .append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
@ -480,7 +471,7 @@ public void testContainerLaunchWithCustomNetworks()
//now set an explicit (non-default) allowedNetwork and ensure that it is //now set an explicit (non-default) allowedNetwork and ensure that it is
// used. // used.
env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_NETWORK, env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK",
customNetwork2); customNetwork2);
runtime.launchContainer(builder.build()); runtime.launchContainer(builder.build());
@ -494,7 +485,6 @@ public void testContainerLaunchWithCustomNetworks()
new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ") new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ")
.append("--workdir=%3$s ") .append("--workdir=%3$s ")
.append("--net=" + customNetwork2 + " ") .append("--net=" + customNetwork2 + " ")
.append("--hostname=" + defaultHostname + " ")
.append(getExpectedTestCapabilitiesArgumentString()) .append(getExpectedTestCapabilitiesArgumentString())
.append(getExpectedCGroupsMountString()) .append(getExpectedCGroupsMountString())
.append("-v %4$s:%4$s ").append("-v %5$s:%5$s ") .append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
@ -515,7 +505,7 @@ public void testContainerLaunchWithCustomNetworks()
//disallowed network should trigger a launch failure //disallowed network should trigger a launch failure
env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_NETWORK, env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK",
customNetwork3); customNetwork3);
try { try {
runtime.launchContainer(builder.build()); runtime.launchContainer(builder.build());
@ -534,8 +524,8 @@ public void testLaunchPrivilegedContainersInvalidEnvVar()
mockExecutor, mockCGroupsHandler); mockExecutor, mockCGroupsHandler);
runtime.initialize(conf); runtime.initialize(conf);
env.put(DockerLinuxContainerRuntime env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER",
.ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER, "invalid-value"); "invalid-value");
runtime.launchContainer(builder.build()); runtime.launchContainer(builder.build());
PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs();
@ -562,8 +552,8 @@ public void testLaunchPrivilegedContainersWithDisabledSetting()
mockExecutor, mockCGroupsHandler); mockExecutor, mockCGroupsHandler);
runtime.initialize(conf); runtime.initialize(conf);
env.put(DockerLinuxContainerRuntime env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER",
.ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER, "true"); "true");
try { try {
runtime.launchContainer(builder.build()); runtime.launchContainer(builder.build());
@ -585,8 +575,8 @@ public void testLaunchPrivilegedContainersWithEnabledSettingAndDefaultACL()
mockExecutor, mockCGroupsHandler); mockExecutor, mockCGroupsHandler);
runtime.initialize(conf); runtime.initialize(conf);
env.put(DockerLinuxContainerRuntime env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER",
.ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER, "true"); "true");
//By default //By default
// yarn.nodemanager.runtime.linux.docker.privileged-containers.acl // yarn.nodemanager.runtime.linux.docker.privileged-containers.acl
// is empty. So we expect this launch to fail. // is empty. So we expect this launch to fail.
@ -615,8 +605,8 @@ public void testLaunchPrivilegedContainersWithEnabledSettingAndDefaultACL()
mockExecutor, mockCGroupsHandler); mockExecutor, mockCGroupsHandler);
runtime.initialize(conf); runtime.initialize(conf);
env.put(DockerLinuxContainerRuntime env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER",
.ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER, "true"); "true");
try { try {
runtime.launchContainer(builder.build()); runtime.launchContainer(builder.build());
@ -642,8 +632,8 @@ public void testLaunchPrivilegedContainersWithEnabledSettingAndDefaultACL()
mockExecutor, mockCGroupsHandler); mockExecutor, mockCGroupsHandler);
runtime.initialize(conf); runtime.initialize(conf);
env.put(DockerLinuxContainerRuntime env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER",
.ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER, "true"); "true");
runtime.launchContainer(builder.build()); runtime.launchContainer(builder.build());
PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs();
@ -937,24 +927,4 @@ public void testDockerImageNamePattern() throws Exception {
} }
} }
} }
@Test
public void testDockerHostnamePattern() throws Exception {
String[] validNames = {"ab", "a.b.c.d", "a1-b.cd.ef", "0AB.", "C_D-"};
String[] invalidNames = {"a", "a#.b.c", "-a.b.c", "a@b.c", "a/b/c"};
for (String name : validNames) {
DockerLinuxContainerRuntime.validateHostname(name);
}
for (String name : invalidNames) {
try {
DockerLinuxContainerRuntime.validateHostname(name);
Assert.fail(name + " is an invalid hostname and should fail the regex");
} catch (ContainerExecutionException ce) {
continue;
}
}
}
} }