diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index d69ae57ca9..95861d7fbf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1882,9 +1882,17 @@ public static boolean isAclEnabled(Configuration conf) { public static final String[] DEFAULT_LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES = {"default"}; + /** Default runtime to be used. */ + public static final String LINUX_CONTAINER_RUNTIME_TYPE = + LINUX_CONTAINER_RUNTIME_PREFIX + "type"; + public static final String DOCKER_CONTAINER_RUNTIME_PREFIX = LINUX_CONTAINER_RUNTIME_PREFIX + "docker."; + /** Default docker image to be used. */ + public static final String NM_DOCKER_IMAGE_NAME = + DOCKER_CONTAINER_RUNTIME_PREFIX + "image-name"; + /** Capabilities allowed (and added by default) for docker containers. **/ public static final String NM_DOCKER_CONTAINER_CAPABILITIES = DOCKER_CONTAINER_RUNTIME_PREFIX + "capabilities"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 0700902e9c..e6f7b37a25 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1725,6 +1725,12 @@ default + + Default container runtime to use. + yarn.nodemanager.runtime.linux.type + + + This configuration setting determines the capabilities assigned to docker containers when they are launched. While these may not @@ -1735,6 +1741,13 @@ CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE + + Default docker image to be used when the docker runtime is + selected. + yarn.nodemanager.runtime.linux.docker.image-name + + + This configuration setting determines if privileged docker containers are allowed on this cluster. Privileged containers are granted diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java index e4cbfdce59..f88dfbfb71 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java @@ -1544,6 +1544,7 @@ public void transition(ContainerImpl container, ContainerEvent event) { // TODO: Add containerWorkDir to the deletion service. if (DockerLinuxContainerRuntime.isDockerContainerRequested( + container.daemonConf, container.getLaunchContext().getEnvironment())) { removeDockerContainer(container); } @@ -1584,6 +1585,7 @@ public void transition(ContainerImpl container, ContainerEvent event) { // TODO: Add containerOuputDir to the deletion service. if (DockerLinuxContainerRuntime.isDockerContainerRequested( + container.daemonConf, container.getLaunchContext().getEnvironment())) { removeDockerContainer(container); } @@ -1858,6 +1860,7 @@ public void transition(ContainerImpl container, ContainerEvent event) { } if (DockerLinuxContainerRuntime.isDockerContainerRequested( + container.daemonConf, container.getLaunchContext().getEnvironment())) { removeDockerContainer(container); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index 6347d4e13e..3fa73ec5ab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -799,6 +799,7 @@ public void cleanupContainer() throws IOException { } // The Docker container may not have fully started, reap the container. if (DockerLinuxContainerRuntime.isDockerContainerRequested( + conf, container.getLaunchContext().getEnvironment())) { reapDockerContainerNoPid(user); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java index 118438296b..e25a6b5bee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java @@ -48,6 +48,7 @@ public class GpuResourceHandlerImpl implements ResourceHandler { public static final String EXCLUDED_GPUS_CLI_OPTION = "--excluded_gpus"; public static final String CONTAINER_ID_CLI_OPTION = "--container_id"; + private Context nmContext; private GpuResourceAllocator gpuAllocator; private CGroupsHandler cGroupsHandler; private PrivilegedOperationExecutor privilegedOperationExecutor; @@ -55,6 +56,7 @@ public class GpuResourceHandlerImpl implements ResourceHandler { public GpuResourceHandlerImpl(Context nmContext, CGroupsHandler cGroupsHandler, PrivilegedOperationExecutor privilegedOperationExecutor) { + this.nmContext = nmContext; this.cGroupsHandler = cGroupsHandler; this.privilegedOperationExecutor = privilegedOperationExecutor; gpuAllocator = new GpuResourceAllocator(nmContext); @@ -102,6 +104,7 @@ public synchronized List preStart(Container container) cGroupsHandler.createCGroup(CGroupsHandler.CGroupController.DEVICES, containerIdStr); if (!DockerLinuxContainerRuntime.isDockerContainerRequested( + nmContext.getConf(), container.getLaunchContext().getEnvironment())) { // Write to devices cgroup only for non-docker container. The reason is // docker engine runtime runc do the devices cgroups initialize in the diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java index 2b8e8adfba..82ca6d91f2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java @@ -24,6 +24,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; @@ -71,7 +72,10 @@ public DefaultLinuxContainerRuntime(PrivilegedOperationExecutor @Override public boolean isRuntimeRequested(Map env) { String type = env.get(ContainerRuntimeConstants.ENV_CONTAINER_TYPE); - return type == null || type.equals("default"); + if (type == null) { + type = conf.get(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE); + } + return type == null || type.isEmpty() || type.equals("default"); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 55dfdc3c24..c86ed53787 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -235,6 +235,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { private Context nmContext; private DockerClient dockerClient; private PrivilegedOperationExecutor privilegedOperationExecutor; + private String defaultImageName; private Set allowedNetworks = new HashSet<>(); private String defaultNetwork; private CGroupsHandler cGroupsHandler; @@ -254,17 +255,17 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { * called {@code YARN_CONTAINER_RUNTIME_TYPE} whose value is {@code docker}, * this method will return true. Otherwise it will return false. * + * @param daemonConf the NodeManager daemon configuration * @param env the environment variable settings for the operation * @return whether a Docker container is requested */ - public static boolean isDockerContainerRequested( + public static boolean isDockerContainerRequested(Configuration daemonConf, Map env) { - if (env == null) { - return false; + String type = (env == null) + ? null : env.get(ContainerRuntimeConstants.ENV_CONTAINER_TYPE); + if (type == null) { + type = daemonConf.get(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE); } - - String type = env.get(ContainerRuntimeConstants.ENV_CONTAINER_TYPE); - return type != null && type.equals("docker"); } @@ -312,6 +313,8 @@ public void initialize(Configuration conf, Context nmContext) defaultROMounts.clear(); defaultRWMounts.clear(); defaultTmpfsMounts.clear(); + defaultImageName = conf.getTrimmed( + YarnConfiguration.NM_DOCKER_IMAGE_NAME, ""); allowedNetworks.addAll(Arrays.asList( conf.getTrimmedStrings( YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_NETWORKS, @@ -325,8 +328,7 @@ public void initialize(Configuration conf, Context nmContext) + " is not in the set of allowed networks: " + allowedNetworks; if (LOG.isWarnEnabled()) { - LOG.warn(message + ". Please check " - + "configuration"); + LOG.warn(message + ". Please check configuration"); } throw new ContainerExecutionException(message); @@ -369,7 +371,7 @@ public void initialize(Configuration conf, Context nmContext) @Override public boolean isRuntimeRequested(Map env) { - return isDockerContainerRequested(env); + return isDockerContainerRequested(conf, env); } private Set getDockerCapabilitiesFromConf() throws @@ -789,6 +791,9 @@ public void launchContainer(ContainerRuntimeContext ctx) String hostname = environment.get(ENV_DOCKER_CONTAINER_HOSTNAME); boolean useEntryPoint = checkUseEntryPoint(environment); + if (imageName == null || imageName.isEmpty()) { + imageName = defaultImageName; + } if(network == null || network.isEmpty()) { network = defaultNetwork; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java index 9a8a4c9f28..301c3e580e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java @@ -80,8 +80,11 @@ public void setup() { mockPrivilegedExecutor = mock(PrivilegedOperationExecutor.class); mockNMStateStore = mock(NMStateStoreService.class); + Configuration conf = new Configuration(); + Context nmctx = mock(Context.class); when(nmctx.getNMStateStore()).thenReturn(mockNMStateStore); + when(nmctx.getConf()).thenReturn(conf); runningContainersMap = new ConcurrentHashMap<>(); when(nmctx.getContainers()).thenReturn(runningContainersMap); @@ -347,15 +350,17 @@ public void testAllocationStored() throws Exception { public void testAllocationStoredWithNULLStateStore() throws Exception { NMNullStateStoreService mockNMNULLStateStore = mock(NMNullStateStoreService.class); + Configuration conf = new YarnConfiguration(); + conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0:0,1:1,2:3,3:4"); + Context nmnctx = mock(Context.class); when(nmnctx.getNMStateStore()).thenReturn(mockNMNULLStateStore); + when(nmnctx.getConf()).thenReturn(conf); GpuResourceHandlerImpl gpuNULLStateResourceHandler = new GpuResourceHandlerImpl(nmnctx, mockCGroupsHandler, mockPrivilegedExecutor); - Configuration conf = new YarnConfiguration(); - conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0:0,1:1,2:3,3:4"); GpuDiscoverer.getInstance().initialize(conf); gpuNULLStateResourceHandler.bootstrap(conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index 4464496e08..a880ba9c88 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -309,11 +309,45 @@ public void testSelectDockerContainerType() { envOtherType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "other"); Assert.assertEquals(false, DockerLinuxContainerRuntime - .isDockerContainerRequested(null)); + .isDockerContainerRequested(conf, null)); Assert.assertEquals(true, DockerLinuxContainerRuntime - .isDockerContainerRequested(envDockerType)); + .isDockerContainerRequested(conf, envDockerType)); Assert.assertEquals(false, DockerLinuxContainerRuntime - .isDockerContainerRequested(envOtherType)); + .isDockerContainerRequested(conf, envOtherType)); + } + + @Test + public void testSelectDockerContainerTypeWithDockerAsDefault() { + Map envDockerType = new HashMap<>(); + Map envOtherType = new HashMap<>(); + + conf.set(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE, "docker"); + envDockerType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "docker"); + envOtherType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "other"); + + Assert.assertEquals(true, DockerLinuxContainerRuntime + .isDockerContainerRequested(conf, null)); + Assert.assertEquals(true, DockerLinuxContainerRuntime + .isDockerContainerRequested(conf, envDockerType)); + Assert.assertEquals(false, DockerLinuxContainerRuntime + .isDockerContainerRequested(conf, envOtherType)); + } + + @Test + public void testSelectDockerContainerTypeWithDefaultSet() { + Map envDockerType = new HashMap<>(); + Map envOtherType = new HashMap<>(); + + conf.set(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE, "default"); + envDockerType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "docker"); + envOtherType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "other"); + + Assert.assertEquals(false, DockerLinuxContainerRuntime + .isDockerContainerRequested(conf, null)); + Assert.assertEquals(true, DockerLinuxContainerRuntime + .isDockerContainerRequested(conf, envDockerType)); + Assert.assertEquals(false, DockerLinuxContainerRuntime + .isDockerContainerRequested(conf, envOtherType)); } private PrivilegedOperation capturePrivilegedOperation() @@ -421,6 +455,57 @@ public void testDockerContainerLaunch() dockerCommands.get(counter)); } + @Test + public void testDockerContainerLaunchWithDefaultImage() + throws ContainerExecutionException, PrivilegedOperationException, + IOException { + conf.set(YarnConfiguration.NM_DOCKER_IMAGE_NAME, "busybox:1.2.3"); + env.remove(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_IMAGE); + + DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( + mockExecutor, mockCGroupsHandler); + runtime.initialize(conf, nmContext); + runtime.launchContainer(builder.build()); + + PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); + List args = op.getArguments(); + String dockerCommandFile = args.get(11); + + List dockerCommands = Files.readAllLines(Paths.get( + dockerCommandFile), Charset.forName("UTF-8")); + + int expected = 13; + int counter = 0; + Assert.assertEquals(expected, dockerCommands.size()); + Assert.assertEquals("[docker-command-execution]", + dockerCommands.get(counter++)); + Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE", + dockerCommands.get(counter++)); + Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++)); + Assert.assertEquals(" detach=true", dockerCommands.get(counter++)); + Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++)); + Assert.assertEquals(" group-add=" + String.join(",", groups), + dockerCommands.get(counter++)); + Assert + .assertEquals(" image=busybox:1.2.3", dockerCommands.get(counter++)); + Assert.assertEquals( + " launch-command=bash,/test_container_work_dir/launch_container.sh", + dockerCommands.get(counter++)); + Assert.assertEquals(" mounts=" + + "/test_container_log_dir:/test_container_log_dir:rw," + + "/test_application_local_dir:/test_application_local_dir:rw," + + "/test_filecache_dir:/test_filecache_dir:ro," + + "/test_user_filecache_dir:/test_user_filecache_dir:ro", + dockerCommands.get(counter++)); + Assert.assertEquals( + " name=container_e11_1518975676334_14532816_01_000001", + dockerCommands.get(counter++)); + Assert.assertEquals(" net=host", dockerCommands.get(counter++)); + Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++)); + Assert.assertEquals(" workdir=/test_container_work_dir", + dockerCommands.get(counter)); + } + @Test public void testContainerLaunchWithUserRemapping() throws ContainerExecutionException, PrivilegedOperationException, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md index 15105f3f00..17a335ef42 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md @@ -114,6 +114,23 @@ The following properties should be set in yarn-site.xml: + + yarn.nodemanager.runtime.linux.type + + + Optional. Sets the default container runtime to use. + + + + + yarn.nodemanager.runtime.linux.docker.image-name + + + Optional. Default docker image to be used when the docker runtime is + selected. + + + yarn.nodemanager.runtime.linux.docker.allowed-container-networks host,none,bridge