From 1388de18ad51434569589a8f5b0b05c38fe02ab3 Mon Sep 17 00:00:00 2001 From: Shane Kumpf Date: Thu, 24 May 2018 09:30:39 -0600 Subject: [PATCH] YARN-6919. Add default volume mount list. Contributed by Eric Badger --- .../hadoop/yarn/conf/YarnConfiguration.java | 10 ++ .../src/main/resources/yarn-default.xml | 14 ++ .../runtime/DockerLinuxContainerRuntime.java | 38 +++++ .../runtime/TestDockerContainerRuntime.java | 138 ++++++++++++++++++ 4 files changed, 200 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 004a59fbba..f7f82f8adc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2002,6 +2002,16 @@ public static boolean isAclEnabled(Configuration conf) { */ public static final int DEFAULT_NM_DOCKER_STOP_GRACE_PERIOD = 10; + /** The default list of read-only mounts to be bind-mounted into all + * Docker containers that use DockerContainerRuntime. */ + public static final String NM_DOCKER_DEFAULT_RO_MOUNTS = + DOCKER_CONTAINER_RUNTIME_PREFIX + "default-ro-mounts"; + + /** The default list of read-write mounts to be bind-mounted into all + * Docker containers that use DockerContainerRuntime. */ + public static final String NM_DOCKER_DEFAULT_RW_MOUNTS = + DOCKER_CONTAINER_RUNTIME_PREFIX + "default-rw-mounts"; + /** The mode in which the Java Container Sandbox should run detailed by * the JavaSandboxLinuxContainerRuntime. */ public static final String YARN_CONTAINER_SANDBOX = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index c82474cfef..b0ffc48e1e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1810,6 +1810,20 @@ 10 + + The default list of read-only mounts to be bind-mounted + into all Docker containers that use DockerContainerRuntime. + yarn.nodemanager.runtime.linux.docker.default-ro-mounts + + + + + The default list of read-write mounts to be bind-mounted + into all Docker containers that use DockerContainerRuntime. + yarn.nodemanager.runtime.linux.docker.default-rw-mounts + + + The mode in which the Java Container Sandbox should run detailed by the JavaSandboxLinuxContainerRuntime. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index e131e9d6c4..5e2233b7d1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -229,6 +229,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { private Set capabilities; private boolean delayedRemovalAllowed; private int dockerStopGracePeriod; + private Set defaultROMounts = new HashSet<>(); + private Set defaultRWMounts = new HashSet<>(); /** * Return whether the given environment variables indicate that the operation @@ -291,6 +293,8 @@ public void initialize(Configuration conf, Context nmContext) this.conf = conf; dockerClient = new DockerClient(conf); allowedNetworks.clear(); + defaultROMounts.clear(); + defaultRWMounts.clear(); allowedNetworks.addAll(Arrays.asList( conf.getTrimmedStrings( YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_NETWORKS, @@ -336,6 +340,14 @@ public void initialize(Configuration conf, Context nmContext) dockerStopGracePeriod = conf.getInt( YarnConfiguration.NM_DOCKER_STOP_GRACE_PERIOD, YarnConfiguration.DEFAULT_NM_DOCKER_STOP_GRACE_PERIOD); + + defaultROMounts.addAll(Arrays.asList( + conf.getTrimmedStrings( + YarnConfiguration.NM_DOCKER_DEFAULT_RO_MOUNTS))); + + defaultRWMounts.addAll(Arrays.asList( + conf.getTrimmedStrings( + YarnConfiguration.NM_DOCKER_DEFAULT_RW_MOUNTS))); } private Set getDockerCapabilitiesFromConf() throws @@ -829,6 +841,32 @@ public void launchContainer(ContainerRuntimeContext ctx) } } + if(defaultROMounts != null && !defaultROMounts.isEmpty()) { + for (String mount : defaultROMounts) { + String[] dir = StringUtils.split(mount, ':'); + if (dir.length != 2) { + throw new ContainerExecutionException("Invalid mount : " + + mount); + } + String src = dir[0]; + String dst = dir[1]; + runCommand.addReadOnlyMountLocation(src, dst); + } + } + + if(defaultRWMounts != null && !defaultRWMounts.isEmpty()) { + for (String mount : defaultRWMounts) { + String[] dir = StringUtils.split(mount, ':'); + if (dir.length != 2) { + throw new ContainerExecutionException("Invalid mount : " + + mount); + } + String src = dir[0]; + String dst = dir[1]; + runCommand.addReadWriteMountLocation(src, dst); + } + } + if (allowHostPidNamespace(container)) { runCommand.setPidNamespace("host"); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index ef21ef093b..b6de366d2e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -82,6 +82,8 @@ import java.util.Set; import java.util.concurrent.ConcurrentMap; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.NM_DOCKER_DEFAULT_RO_MOUNTS; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.NM_DOCKER_DEFAULT_RW_MOUNTS; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.APPID; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.APPLICATION_LOCAL_DIRS; @@ -1331,6 +1333,142 @@ public void testUserMountModeNulInvalid() throws ContainerExecutionException { } } + @Test + public void testDefaultROMounts() + throws ContainerExecutionException, PrivilegedOperationException, + IOException { + conf.setStrings(NM_DOCKER_DEFAULT_RO_MOUNTS, + "/tmp/foo:/tmp/foo,/tmp/bar:/tmp/bar"); + DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( + mockExecutor, mockCGroupsHandler); + runtime.initialize(conf, nmContext); + + runtime.launchContainer(builder.build()); + PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); + List args = op.getArguments(); + String dockerCommandFile = args.get(11); + + List dockerCommands = Files.readAllLines( + Paths.get(dockerCommandFile), Charset.forName("UTF-8")); + + int expected = 14; + int counter = 0; + Assert.assertEquals(expected, dockerCommands.size()); + Assert.assertEquals("[docker-command-execution]", + dockerCommands.get(counter++)); + Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE", + dockerCommands.get(counter++)); + Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++)); + Assert.assertEquals(" detach=true", dockerCommands.get(counter++)); + Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++)); + Assert.assertEquals(" group-add=" + String.join(",", groups), + dockerCommands.get(counter++)); + Assert.assertEquals(" image=busybox:latest", + dockerCommands.get(counter++)); + Assert.assertEquals( + " launch-command=bash,/test_container_work_dir/launch_container.sh", + dockerCommands.get(counter++)); + Assert.assertEquals( + " name=container_e11_1518975676334_14532816_01_000001", + dockerCommands.get(counter++)); + Assert.assertEquals(" net=host", dockerCommands.get(counter++)); + Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir," + + "/test_user_filecache_dir:/test_user_filecache_dir," + + "/tmp/foo:/tmp/foo,/tmp/bar:/tmp/bar", + dockerCommands.get(counter++)); + Assert.assertEquals( + " rw-mounts=/test_container_log_dir:/test_container_log_dir," + + "/test_application_local_dir:/test_application_local_dir", + dockerCommands.get(counter++)); + Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++)); + Assert.assertEquals(" workdir=/test_container_work_dir", + dockerCommands.get(counter)); + } + + @Test + public void testDefaultROMountsInvalid() throws ContainerExecutionException { + conf.setStrings(NM_DOCKER_DEFAULT_RO_MOUNTS, + "source,target"); + DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( + mockExecutor, mockCGroupsHandler); + runtime.initialize(conf, nmContext); + + try { + runtime.launchContainer(builder.build()); + Assert.fail("Expected a launch container failure due to invalid mount."); + } catch (ContainerExecutionException e) { + LOG.info("Caught expected exception : " + e); + } + } + + @Test + public void testDefaultRWMounts() + throws ContainerExecutionException, PrivilegedOperationException, + IOException { + conf.setStrings(NM_DOCKER_DEFAULT_RW_MOUNTS, + "/tmp/foo:/tmp/foo,/tmp/bar:/tmp/bar"); + DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( + mockExecutor, mockCGroupsHandler); + runtime.initialize(conf, nmContext); + + runtime.launchContainer(builder.build()); + PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); + List args = op.getArguments(); + String dockerCommandFile = args.get(11); + + List dockerCommands = Files.readAllLines( + Paths.get(dockerCommandFile), Charset.forName("UTF-8")); + + int expected = 14; + int counter = 0; + Assert.assertEquals(expected, dockerCommands.size()); + Assert.assertEquals("[docker-command-execution]", + dockerCommands.get(counter++)); + Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE", + dockerCommands.get(counter++)); + Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++)); + Assert.assertEquals(" detach=true", dockerCommands.get(counter++)); + Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++)); + Assert.assertEquals(" group-add=" + String.join(",", groups), + dockerCommands.get(counter++)); + Assert.assertEquals(" image=busybox:latest", + dockerCommands.get(counter++)); + Assert.assertEquals( + " launch-command=bash,/test_container_work_dir/launch_container.sh", + dockerCommands.get(counter++)); + Assert.assertEquals( + " name=container_e11_1518975676334_14532816_01_000001", + dockerCommands.get(counter++)); + Assert.assertEquals(" net=host", dockerCommands.get(counter++)); + Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir," + + "/test_user_filecache_dir:/test_user_filecache_dir", + dockerCommands.get(counter++)); + Assert.assertEquals( + " rw-mounts=/test_container_log_dir:/test_container_log_dir," + + "/test_application_local_dir:/test_application_local_dir," + + "/tmp/foo:/tmp/foo,/tmp/bar:/tmp/bar", + dockerCommands.get(counter++)); + Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++)); + Assert.assertEquals(" workdir=/test_container_work_dir", + dockerCommands.get(counter)); + } + + @Test + public void testDefaultRWMountsInvalid() throws ContainerExecutionException { + conf.setStrings(NM_DOCKER_DEFAULT_RW_MOUNTS, + "source,target"); + DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( + mockExecutor, mockCGroupsHandler); + runtime.initialize(conf, nmContext); + + try { + runtime.launchContainer(builder.build()); + Assert.fail("Expected a launch container failure due to invalid mount."); + } catch (ContainerExecutionException e) { + LOG.info("Caught expected exception : " + e); + } + } + @Test public void testContainerLivelinessCheck() throws ContainerExecutionException, PrivilegedOperationException {