big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-5534. Allow user provided Docker volume mount list. (Contributed by Shane Kumpf)

Browse Source
This commit is contained in:
Eric Yang 2017-11-22 13:05:34 -05:00
parent de8b6ca5ef
commit d42a336cfa
4 changed files with 211 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
View File

@ -65,6 +65,7 @@
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*;
@ -134,6 +135,16 @@
* source is an absolute path that is not a symlink and that points to a
* localized resource.
* </li>
* <li>
* {@code YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS} allows users to specify
+ additional volume mounts for the Docker container. The value of the
* environment variable should be a comma-separated list of mounts.
* All such mounts must be given as {@code source:dest:mode}, and the mode
* must be "ro" (read-only) or "rw" (read-write) to specify the type of
* access being requested. The requested mounts will be validated by
* container-executor based on the values set in container-executor.cfg for
* {@code docker.allowed.ro-mounts} and {@code docker.allowed.rw-mounts}.
* </li>
* </ul>
*/
@InterfaceAudience.Private
@ -151,6 +162,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
"^[a-zA-Z0-9][a-zA-Z0-9_.-]+$";
private static final Pattern hostnamePattern = Pattern.compile(
HOSTNAME_PATTERN);
private static final Pattern USER_MOUNT_PATTERN = Pattern.compile(
"(?<=^|,)([^:\\x00]+):([^:\\x00]+):([a-z]+)");
@InterfaceAudience.Private
public static final String ENV_DOCKER_CONTAINER_IMAGE =
@ -176,6 +189,9 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
@InterfaceAudience.Private
public static final String ENV_DOCKER_CONTAINER_LOCAL_RESOURCE_MOUNTS =
"YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS";
@InterfaceAudience.Private
public static final String ENV_DOCKER_CONTAINER_MOUNTS =
"YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS";
private Configuration conf;
private Context nmContext;
@ -675,6 +691,32 @@ public void launchContainer(ContainerRuntimeContext ctx)
}
}
if (environment.containsKey(ENV_DOCKER_CONTAINER_MOUNTS)) {
Matcher parsedMounts = USER_MOUNT_PATTERN.matcher(
environment.get(ENV_DOCKER_CONTAINER_MOUNTS));
if (!parsedMounts.find()) {
throw new ContainerExecutionException(
"Unable to parse user supplied mount list: "
+ environment.get(ENV_DOCKER_CONTAINER_MOUNTS));
}
parsedMounts.reset();
while (parsedMounts.find()) {
String src = parsedMounts.group(1);
String dst = parsedMounts.group(2);
String mode = parsedMounts.group(3);
if (!mode.equals("ro") && !mode.equals("rw")) {
throw new ContainerExecutionException(
"Invalid mount mode requested for mount: "
+ parsedMounts.group());
}
if (mode.equals("ro")) {
runCommand.addReadOnlyMountLocation(src, dst);
} else {
runCommand.addReadWriteMountLocation(src, dst);
}
}
}
if (allowPrivilegedContainerExecution(container)) {
runCommand.setPrivileged();
}

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
View File

@ -66,6 +66,12 @@ public DockerRunCommand addMountLocation(String sourcePath, String
return this;
}
public DockerRunCommand addReadWriteMountLocation(String sourcePath, String
destinationPath) {
super.addCommandArguments("rw-mounts", sourcePath + ":" + destinationPath);
return this;
}
public DockerRunCommand addReadOnlyMountLocation(String sourcePath, String
destinationPath, boolean createSource) {
boolean sourceExists = new File(sourcePath).exists();
@ -76,6 +82,12 @@ public DockerRunCommand addReadOnlyMountLocation(String sourcePath, String
return this;
}
public DockerRunCommand addReadOnlyMountLocation(String sourcePath, String
destinationPath) {
super.addCommandArguments("ro-mounts", sourcePath + ":" + destinationPath);
return this;
}
public DockerRunCommand setVolumeDriver(String volumeDriver) {
super.addCommandArguments("volume-driver", volumeDriver);
return this;

109
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
View File

@ -1035,6 +1035,115 @@ public void testMountMultiple()
}
@Test
public void testUserMounts()
throws ContainerExecutionException, PrivilegedOperationException,
IOException{
DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
mockExecutor, mockCGroupsHandler);
runtime.initialize(conf, null);
env.put(
DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_MOUNTS,
"/tmp/foo:/tmp/foo:ro,/tmp/bar:/tmp/bar:rw");
runtime.launchContainer(builder.build());
PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs();
List<String> args = op.getArguments();
String dockerCommandFile = args.get(11);
List<String> dockerCommands = Files.readAllLines(
Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
Assert.assertEquals(14, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]", dockerCommands.get(0));
Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
dockerCommands.get(1));
Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(2));
Assert.assertEquals(" detach=true", dockerCommands.get(3));
Assert.assertEquals(" docker-command=run", dockerCommands.get(4));
Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(5));
Assert.assertEquals(" image=busybox:latest", dockerCommands.get(6));
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
dockerCommands.get(7));
Assert.assertEquals(" name=container_id", dockerCommands.get(8));
Assert.assertEquals(" net=host", dockerCommands.get(9));
Assert.assertEquals(" ro-mounts=/tmp/foo:/tmp/foo",
dockerCommands.get(10));
Assert.assertEquals(
" rw-mounts=/test_container_local_dir:/test_container_local_dir,"
+ "/test_filecache_dir:/test_filecache_dir,"
+ "/test_container_work_dir:/test_container_work_dir,"
+ "/test_container_log_dir:/test_container_log_dir,"
+ "/test_user_local_dir:/test_user_local_dir,"
+ "/tmp/bar:/tmp/bar",
dockerCommands.get(11));
Assert.assertEquals(" user=run_as_user", dockerCommands.get(12));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(13));
}
@Test
public void testUserMountInvalid()
throws ContainerExecutionException, PrivilegedOperationException,
IOException{
DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
mockExecutor, mockCGroupsHandler);
runtime.initialize(conf, null);
env.put(
DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_MOUNTS,
"source:target");
try {
runtime.launchContainer(builder.build());
Assert.fail("Expected a launch container failure due to invalid mount.");
} catch (ContainerExecutionException e) {
LOG.info("Caught expected exception : " + e);
}
}
@Test
public void testUserMountModeInvalid()
throws ContainerExecutionException, PrivilegedOperationException,
IOException{
DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
mockExecutor, mockCGroupsHandler);
runtime.initialize(conf, null);
env.put(
DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_MOUNTS,
"source:target:other");
try {
runtime.launchContainer(builder.build());
Assert.fail("Expected a launch container failure due to invalid mode.");
} catch (ContainerExecutionException e) {
LOG.info("Caught expected exception : " + e);
}
}
@Test
public void testUserMountModeNulInvalid()
throws ContainerExecutionException, PrivilegedOperationException,
IOException{
DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
mockExecutor, mockCGroupsHandler);
runtime.initialize(conf, null);
env.put(
DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_MOUNTS,
"s\0ource:target:ro");
try {
runtime.launchContainer(builder.build());
Assert.fail("Expected a launch container failure due to NUL in mount.");
} catch (ContainerExecutionException e) {
LOG.info("Caught expected exception : " + e);
}
}
@Test
public void testContainerLivelinessCheck()
throws ContainerExecutionException, PrivilegedOperationException {

48
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
View File

@ -290,6 +290,7 @@ environment variables in the application's environment:
| `YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK` | Sets the network type to be used by the Docker container. It must be a valid value as determined by the yarn.nodemanager.runtime.linux.docker.allowed-container-networks property. |
| `YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER` | Controls whether the Docker container is a privileged container. In order to use privileged containers, the yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed property must be set to true, and the application owner must appear in the value of the yarn.nodemanager.runtime.linux.docker.privileged-containers.acl property. If this environment variable is set to true, a privileged Docker container will be used if allowed. No other value is allowed, so the environment variable should be left unset rather than setting it to false. |
| `YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS` | Adds additional volume mounts to the Docker container. The value of the environment variable should be a comma-separated list of mounts. All such mounts must be given as "source:dest", where the source is an absolute path that is not a symlink and that points to a localized resource. Note that as of YARN-5298, localized directories are automatically mounted into the container as volumes. |
| `YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS` | Adds additional volume mounts to the Docker container. The value of the environment variable should be a comma-separated list of mounts. All such mounts must be given as "source:dest:mode" and the mode must be "ro" (read-only) or "rw" (read-write) to specify the type of access being requested. The requested mounts will be validated by container-executor based on the values set in container-executor.cfg for docker.allowed.ro-mounts and docker.allowed.rw-mounts. |
The first two are required. The remainder can be set as needed. While
controlling the container type through environment variables is somewhat less
@ -302,6 +303,53 @@ the application will behave exactly as any other YARN application. Logs will be
aggregated and stored in the relevant history server. The application life cycle
will be the same as for a non-Docker application.
Using Docker Bind Mounted Volumes
---------------------------------
**WARNING** Care should be taken when enabling this feature. Enabling access to
directories such as, but not limited to, /, /etc, /run, or /home is not
advisable and can result in containers negatively impacting the host or leaking
sensitive information. **WARNING**
Files and directories from the host are commonly needed within the Docker
containers, which Docker provides through
[volumes](https://docs.docker.com/engine/tutorials/dockervolumes/).
Examples include localized resources, Apache Hadoop binaries, and sockets. To
facilitate this need, YARN-6623 added the ability for administrators to set a
whitelist of host directories that are allowed to be bind mounted as volumes
into containers. YARN-5534 added the ability for users to supply a list of
mounts that will be mounted into the containers, if allowed by the
administrative whitelist.
In order to make use of this feature, the following must be configured.
* The administrator must define the volume whitelist in container-executor.cfg by setting `docker.allowed.ro-mounts` and `docker.allowed.rw-mounts` to the list of parent directories that are allowed to be mounted.
* The application submitter requests the required volumes at application submission time using the `YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS` environment variable.
The administrator supplied whitelist is defined as a comma separated list of
directories that are allowed to be mounted into containers. The source directory
supplied by the user must either match or be a child of the specified
directory.
The user supplied mount list is defined as a comma separated list in the form
*source*:*destination*:*mode*. The source is the file or directory on the host.
The destination is the path within the contatiner where the source will be bind
mounted. The mode defines the mode the user expects for the mount, which can be
ro (read-only) or rw (read-write).
The following example outlines how to use this feature to mount the commonly
needed /sys/fs/cgroup directory into the container running on YARN.
The administrator sets docker.allowed.ro-mounts in container-executor.cfg to
"/sys/fs/cgroup". Applications can now request that "/sys/fs/cgroup" be mounted
from the host into the container in read-only mode.
At application submission time, the YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS
environment variable can then be set to request this mount. In this example,
the environment variable would be set to "/sys/fs/cgroup:/sys/fs/cgroup:ro".
The destination path is not restricted, "/sys/fs/cgroup:/cgroup:ro" would also
be valid given the example admin whitelist.
Connecting to a Secure Docker Repository
----------------------------------------