diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 34f1e935dd..860227e216 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2030,6 +2030,16 @@ public static boolean isAclEnabled(Configuration conf) { public static final String DEFAULT_NM_DOCKER_DEFAULT_CONTAINER_NETWORK = "host"; + /** The set of runtimes allowed when launching containers using the + * DockerContainerRuntime. */ + public static final String NM_DOCKER_ALLOWED_CONTAINER_RUNTIMES = + DOCKER_CONTAINER_RUNTIME_PREFIX + "allowed-container-runtimes"; + + /** The set of runtimes allowed when launching containers using the + * DockerContainerRuntime. */ + public static final String[] DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_RUNTIMES = + {"runc"}; + /** Allow host pid namespace for containers. Use with care. */ public static final String NM_DOCKER_ALLOW_HOST_PID_NAMESPACE = DOCKER_CONTAINER_RUNTIME_PREFIX + "host-pid-namespace.allowed"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 004af7c394..e8659429b6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1803,6 +1803,13 @@ host + + The set of runtimes allowed when launching containers using the + DockerContainerRuntime. + yarn.nodemanager.runtime.linux.docker.allowed-container-runtimes + runc + + This configuration setting determines whether the host's PID namespace is allowed for docker containers on this cluster. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 384bc5e0e2..85ddca90da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -265,6 +265,9 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { @InterfaceAudience.Private public static final String ENV_DOCKER_CONTAINER_YARN_SYSFS = "YARN_CONTAINER_RUNTIME_YARN_SYSFS_ENABLE"; + @InterfaceAudience.Private + public static final String ENV_DOCKER_CONTAINER_DOCKER_RUNTIME = + "YARN_CONTAINER_RUNTIME_DOCKER_RUNTIME"; public static final String YARN_SYSFS_PATH = "/hadoop/yarn/sysfs"; private Configuration conf; @@ -275,7 +278,9 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { private String defaultImageName; private Boolean defaultImageUpdate; private Set allowedNetworks = new HashSet<>(); + private Set allowedRuntimes = new HashSet<>(); private String defaultNetwork; + private String defaultRuntime; private CGroupsHandler cGroupsHandler; private AccessControlList privilegedContainersAcl; private boolean enableUserReMapping; @@ -349,6 +354,7 @@ public void initialize(Configuration conf, Context nmContext) this.conf = conf; dockerClient = new DockerClient(); allowedNetworks.clear(); + allowedRuntimes.clear(); defaultROMounts.clear(); defaultRWMounts.clear(); defaultTmpfsMounts.clear(); @@ -363,6 +369,10 @@ public void initialize(Configuration conf, Context nmContext) defaultNetwork = conf.getTrimmed( YarnConfiguration.NM_DOCKER_DEFAULT_CONTAINER_NETWORK, YarnConfiguration.DEFAULT_NM_DOCKER_DEFAULT_CONTAINER_NETWORK); + allowedRuntimes.addAll(Arrays.asList( + conf.getTrimmedStrings( + YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_RUNTIMES, + YarnConfiguration.DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_RUNTIMES))); if(!allowedNetworks.contains(defaultNetwork)) { String message = "Default network: " + defaultNetwork @@ -529,6 +539,19 @@ private void validateContainerNetworkType(String network) throw new ContainerExecutionException(msg); } + private void validateContainerRuntimeType(String runtime) + throws ContainerExecutionException { + if (runtime == null || runtime.isEmpty() + || allowedRuntimes.contains(runtime)) { + return; + } + + String msg = "Disallowed runtime: '" + runtime + + "' specified. Allowed networks: are " + allowedRuntimes + .toString(); + throw new ContainerExecutionException(msg); + } + /** * Return whether the YARN container is allowed to run using the host's PID * namespace for the Docker container. For this to be allowed, the submitting @@ -801,6 +824,7 @@ public void launchContainer(ContainerRuntimeContext ctx) String imageName = environment.get(ENV_DOCKER_CONTAINER_IMAGE); String network = environment.get(ENV_DOCKER_CONTAINER_NETWORK); String hostname = environment.get(ENV_DOCKER_CONTAINER_HOSTNAME); + String runtime = environment.get(ENV_DOCKER_CONTAINER_DOCKER_RUNTIME); boolean useEntryPoint = checkUseEntryPoint(environment); if (imageName == null || imageName.isEmpty()) { @@ -816,6 +840,8 @@ public void launchContainer(ContainerRuntimeContext ctx) validateImageName(imageName); + validateContainerRuntimeType(runtime); + if (defaultImageUpdate) { pullImageFromRemote(containerIdStr, imageName); } @@ -886,6 +912,9 @@ public void launchContainer(ContainerRuntimeContext ctx) } runCommand.setCapabilities(capabilities); + if (runtime != null && !runtime.isEmpty()) { + runCommand.addRuntime(runtime); + } runCommand.addAllReadWriteMountLocations(containerLogDirs); runCommand.addAllReadWriteMountLocations(applicationLocalDirs); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index 6669cac6d4..5d4d22ebd2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -2481,6 +2481,56 @@ public void testDockerContainerRelaunch() dockerCommands.get(counter)); } + @Test + public void testLaunchContainersWithSpecificDockerRuntime() + throws ContainerExecutionException, PrivilegedOperationException, + IOException { + DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( + mockExecutor, mockCGroupsHandler); + runtime.initialize(conf, nmContext); + + env.put(DockerLinuxContainerRuntime + .ENV_DOCKER_CONTAINER_DOCKER_RUNTIME, "runc"); + runtime.launchContainer(builder.build()); + List dockerCommands = readDockerCommands(); + Assert.assertEquals(14, dockerCommands.size()); + Assert.assertEquals(" runtime=runc", dockerCommands.get(11)); + } + + @Test + @SuppressWarnings("unchecked") + public void testContainerLaunchWithAllowedRuntimes() + throws ContainerExecutionException, IOException, + PrivilegedOperationException { + DockerLinuxContainerRuntime runtime = + new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler); + runtime.initialize(conf, nmContext); + + String disallowedRuntime = "runc2"; + + try { + env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_DOCKER_RUNTIME, + disallowedRuntime); + runtime.launchContainer(builder.build()); + Assert.fail("Runtime was expected to be disallowed: " + + disallowedRuntime); + } catch (ContainerExecutionException e) { + LOG.info("Caught expected exception: " + e); + } + + String allowedRuntime = "runc"; + env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_DOCKER_RUNTIME, + allowedRuntime); + //this should cause no failures. + + runtime.launchContainer(builder.build()); + List dockerCommands = readDockerCommands(); + + //This is the expected docker invocation for this case + Assert.assertEquals(14, dockerCommands.size()); + Assert.assertEquals(" runtime=runc", dockerCommands.get(11)); + } + private static void verifyStopCommand(List dockerCommands, String signal) { Assert.assertEquals(4, dockerCommands.size());