big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-6456. Added config to set default container runtimes.

Browse Source 
Contributed by Craig Condit
This commit is contained in:
Eric Yang 2018-09-27 15:28:38 -04:00
parent 184544eff8
commit b237a0dd44
10 changed files with 159 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -1882,9 +1882,17 @@ public static boolean isAclEnabled(Configuration conf) {
public static final String[] DEFAULT_LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES
= {"default"};
/** Default runtime to be used. */
public static final String LINUX_CONTAINER_RUNTIME_TYPE =
LINUX_CONTAINER_RUNTIME_PREFIX + "type";
public static final String DOCKER_CONTAINER_RUNTIME_PREFIX =
LINUX_CONTAINER_RUNTIME_PREFIX + "docker.";
/** Default docker image to be used. */
public static final String NM_DOCKER_IMAGE_NAME =
DOCKER_CONTAINER_RUNTIME_PREFIX + "image-name";
/** Capabilities allowed (and added by default) for docker containers. **/
public static final String NM_DOCKER_CONTAINER_CAPABILITIES =
DOCKER_CONTAINER_RUNTIME_PREFIX + "capabilities";

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -1725,6 +1725,12 @@
<value>default</value>
</property>
<property>
<description>Default container runtime to use.</description>
<name>yarn.nodemanager.runtime.linux.type</name>
<value></value>
</property>
<property>
<description>This configuration setting determines the capabilities
assigned to docker containers when they are launched. While these may not
@ -1735,6 +1741,13 @@
<value>CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE</value>
</property>
<property>
<description>Default docker image to be used when the docker runtime is
selected.</description>
<name>yarn.nodemanager.runtime.linux.docker.image-name</name>
<value></value>
</property>
<property>
<description>This configuration setting determines if privileged docker
containers are allowed on this cluster. Privileged containers are granted

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
View File

@ -1544,6 +1544,7 @@ public void transition(ContainerImpl container, ContainerEvent event) {
// TODO: Add containerWorkDir to the deletion service.
if (DockerLinuxContainerRuntime.isDockerContainerRequested(
container.daemonConf,
container.getLaunchContext().getEnvironment())) {
removeDockerContainer(container);
}
@ -1584,6 +1585,7 @@ public void transition(ContainerImpl container, ContainerEvent event) {
// TODO: Add containerOuputDir to the deletion service.
if (DockerLinuxContainerRuntime.isDockerContainerRequested(
container.daemonConf,
container.getLaunchContext().getEnvironment())) {
removeDockerContainer(container);
}
@ -1858,6 +1860,7 @@ public void transition(ContainerImpl container, ContainerEvent event) {
}
if (DockerLinuxContainerRuntime.isDockerContainerRequested(
container.daemonConf,
container.getLaunchContext().getEnvironment())) {
removeDockerContainer(container);
}

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
View File

@ -799,6 +799,7 @@ public void cleanupContainer() throws IOException {
}
// The Docker container may not have fully started, reap the container.
if (DockerLinuxContainerRuntime.isDockerContainerRequested(
conf,
container.getLaunchContext().getEnvironment())) {
reapDockerContainerNoPid(user);
}

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
View File

@ -48,6 +48,7 @@ public class GpuResourceHandlerImpl implements ResourceHandler {
public static final String EXCLUDED_GPUS_CLI_OPTION = "--excluded_gpus";
public static final String CONTAINER_ID_CLI_OPTION = "--container_id";
private Context nmContext;
private GpuResourceAllocator gpuAllocator;
private CGroupsHandler cGroupsHandler;
private PrivilegedOperationExecutor privilegedOperationExecutor;
@ -55,6 +56,7 @@ public class GpuResourceHandlerImpl implements ResourceHandler {
public GpuResourceHandlerImpl(Context nmContext,
CGroupsHandler cGroupsHandler,
PrivilegedOperationExecutor privilegedOperationExecutor) {
this.nmContext = nmContext;
this.cGroupsHandler = cGroupsHandler;
this.privilegedOperationExecutor = privilegedOperationExecutor;
gpuAllocator = new GpuResourceAllocator(nmContext);
@ -102,6 +104,7 @@ public synchronized List<PrivilegedOperation> preStart(Container container)
cGroupsHandler.createCGroup(CGroupsHandler.CGroupController.DEVICES,
containerIdStr);
if (!DockerLinuxContainerRuntime.isDockerContainerRequested(
nmContext.getConf(),
container.getLaunchContext().getEnvironment())) {
// Write to devices cgroup only for non-docker container. The reason is
// docker engine runtime runc do the devices cgroups initialize in the

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
View File

@ -24,6 +24,7 @@
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@ -71,7 +72,10 @@ public DefaultLinuxContainerRuntime(PrivilegedOperationExecutor
@Override
public boolean isRuntimeRequested(Map<String, String> env) {
String type = env.get(ContainerRuntimeConstants.ENV_CONTAINER_TYPE);
return type == null || type.equals("default");
if (type == null) {
type = conf.get(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE);
}
return type == null || type.isEmpty() || type.equals("default");
}
@Override

23
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
View File

@ -235,6 +235,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
private Context nmContext;
private DockerClient dockerClient;
private PrivilegedOperationExecutor privilegedOperationExecutor;
private String defaultImageName;
private Set<String> allowedNetworks = new HashSet<>();
private String defaultNetwork;
private CGroupsHandler cGroupsHandler;
@ -254,17 +255,17 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
* called {@code YARN_CONTAINER_RUNTIME_TYPE} whose value is {@code docker},
* this method will return true. Otherwise it will return false.
*
* @param daemonConf the NodeManager daemon configuration
* @param env the environment variable settings for the operation
* @return whether a Docker container is requested
*/
public static boolean isDockerContainerRequested(
public static boolean isDockerContainerRequested(Configuration daemonConf,
Map<String, String> env) {
if (env == null) {
return false;
String type = (env == null)
? null : env.get(ContainerRuntimeConstants.ENV_CONTAINER_TYPE);
if (type == null) {
type = daemonConf.get(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE);
}
String type = env.get(ContainerRuntimeConstants.ENV_CONTAINER_TYPE);
return type != null && type.equals("docker");
}
@ -312,6 +313,8 @@ public void initialize(Configuration conf, Context nmContext)
defaultROMounts.clear();
defaultRWMounts.clear();
defaultTmpfsMounts.clear();
defaultImageName = conf.getTrimmed(
YarnConfiguration.NM_DOCKER_IMAGE_NAME, "");
allowedNetworks.addAll(Arrays.asList(
conf.getTrimmedStrings(
YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_NETWORKS,
@ -325,8 +328,7 @@ public void initialize(Configuration conf, Context nmContext)
+ " is not in the set of allowed networks: " + allowedNetworks;
if (LOG.isWarnEnabled()) {
LOG.warn(message + ". Please check "
+ "configuration");
LOG.warn(message + ". Please check configuration");
}
throw new ContainerExecutionException(message);
@ -369,7 +371,7 @@ public void initialize(Configuration conf, Context nmContext)
@Override
public boolean isRuntimeRequested(Map<String, String> env) {
return isDockerContainerRequested(env);
return isDockerContainerRequested(conf, env);
}
private Set<String> getDockerCapabilitiesFromConf() throws
@ -789,6 +791,9 @@ public void launchContainer(ContainerRuntimeContext ctx)
String hostname = environment.get(ENV_DOCKER_CONTAINER_HOSTNAME);
boolean useEntryPoint = checkUseEntryPoint(environment);
if (imageName == null || imageName.isEmpty()) {
imageName = defaultImageName;
}
if(network == null || network.isEmpty()) {
network = defaultNetwork;
}

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
View File

@ -80,8 +80,11 @@ public void setup() {
mockPrivilegedExecutor = mock(PrivilegedOperationExecutor.class);
mockNMStateStore = mock(NMStateStoreService.class);
Configuration conf = new Configuration();
Context nmctx = mock(Context.class);
when(nmctx.getNMStateStore()).thenReturn(mockNMStateStore);
when(nmctx.getConf()).thenReturn(conf);
runningContainersMap = new ConcurrentHashMap<>();
when(nmctx.getContainers()).thenReturn(runningContainersMap);
@ -347,15 +350,17 @@ public void testAllocationStored() throws Exception {
public void testAllocationStoredWithNULLStateStore() throws Exception {
NMNullStateStoreService mockNMNULLStateStore = mock(NMNullStateStoreService.class);
Configuration conf = new YarnConfiguration();
conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0:0,1:1,2:3,3:4");
Context nmnctx = mock(Context.class);
when(nmnctx.getNMStateStore()).thenReturn(mockNMNULLStateStore);
when(nmnctx.getConf()).thenReturn(conf);
GpuResourceHandlerImpl gpuNULLStateResourceHandler =
new GpuResourceHandlerImpl(nmnctx, mockCGroupsHandler,
mockPrivilegedExecutor);
Configuration conf = new YarnConfiguration();
conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0:0,1:1,2:3,3:4");
GpuDiscoverer.getInstance().initialize(conf);
gpuNULLStateResourceHandler.bootstrap(conf);

91
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
View File

@ -309,11 +309,45 @@ public void testSelectDockerContainerType() {
envOtherType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "other");
Assert.assertEquals(false, DockerLinuxContainerRuntime
.isDockerContainerRequested(null));
.isDockerContainerRequested(conf, null));
Assert.assertEquals(true, DockerLinuxContainerRuntime
.isDockerContainerRequested(envDockerType));
.isDockerContainerRequested(conf, envDockerType));
Assert.assertEquals(false, DockerLinuxContainerRuntime
.isDockerContainerRequested(envOtherType));
.isDockerContainerRequested(conf, envOtherType));
}
@Test
public void testSelectDockerContainerTypeWithDockerAsDefault() {
Map<String, String> envDockerType = new HashMap<>();
Map<String, String> envOtherType = new HashMap<>();
conf.set(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE, "docker");
envDockerType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "docker");
envOtherType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "other");
Assert.assertEquals(true, DockerLinuxContainerRuntime
.isDockerContainerRequested(conf, null));
Assert.assertEquals(true, DockerLinuxContainerRuntime
.isDockerContainerRequested(conf, envDockerType));
Assert.assertEquals(false, DockerLinuxContainerRuntime
.isDockerContainerRequested(conf, envOtherType));
}
@Test
public void testSelectDockerContainerTypeWithDefaultSet() {
Map<String, String> envDockerType = new HashMap<>();
Map<String, String> envOtherType = new HashMap<>();
conf.set(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE, "default");
envDockerType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "docker");
envOtherType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "other");
Assert.assertEquals(false, DockerLinuxContainerRuntime
.isDockerContainerRequested(conf, null));
Assert.assertEquals(true, DockerLinuxContainerRuntime
.isDockerContainerRequested(conf, envDockerType));
Assert.assertEquals(false, DockerLinuxContainerRuntime
.isDockerContainerRequested(conf, envOtherType));
}
private PrivilegedOperation capturePrivilegedOperation()
@ -421,6 +455,57 @@ public void testDockerContainerLaunch()
dockerCommands.get(counter));
}
@Test
public void testDockerContainerLaunchWithDefaultImage()
throws ContainerExecutionException, PrivilegedOperationException,
IOException {
conf.set(YarnConfiguration.NM_DOCKER_IMAGE_NAME, "busybox:1.2.3");
env.remove(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_IMAGE);
DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
mockExecutor, mockCGroupsHandler);
runtime.initialize(conf, nmContext);
runtime.launchContainer(builder.build());
PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs();
List<String> args = op.getArguments();
String dockerCommandFile = args.get(11);
List<String> dockerCommands = Files.readAllLines(Paths.get(
dockerCommandFile), Charset.forName("UTF-8"));
int expected = 13;
int counter = 0;
Assert.assertEquals(expected, dockerCommands.size());
Assert.assertEquals("[docker-command-execution]",
dockerCommands.get(counter++));
Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
dockerCommands.get(counter++));
Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++));
Assert.assertEquals(" detach=true", dockerCommands.get(counter++));
Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++));
Assert.assertEquals(" group-add=" + String.join(",", groups),
dockerCommands.get(counter++));
Assert
.assertEquals(" image=busybox:1.2.3", dockerCommands.get(counter++));
Assert.assertEquals(
" launch-command=bash,/test_container_work_dir/launch_container.sh",
dockerCommands.get(counter++));
Assert.assertEquals(" mounts="
+ "/test_container_log_dir:/test_container_log_dir:rw,"
+ "/test_application_local_dir:/test_application_local_dir:rw,"
+ "/test_filecache_dir:/test_filecache_dir:ro,"
+ "/test_user_filecache_dir:/test_user_filecache_dir:ro",
dockerCommands.get(counter++));
Assert.assertEquals(
" name=container_e11_1518975676334_14532816_01_000001",
dockerCommands.get(counter++));
Assert.assertEquals(" net=host", dockerCommands.get(counter++));
Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++));
Assert.assertEquals(" workdir=/test_container_work_dir",
dockerCommands.get(counter));
}
@Test
public void testContainerLaunchWithUserRemapping()
throws ContainerExecutionException, PrivilegedOperationException,

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
View File

@ -114,6 +114,23 @@ The following properties should be set in yarn-site.xml:
</description>
</property>
<property>
<name>yarn.nodemanager.runtime.linux.type</name>
<value></value>
<description>
Optional. Sets the default container runtime to use.
</description>
</property>
<property>
<name>yarn.nodemanager.runtime.linux.docker.image-name</name>
<value></value>
<description>
Optional. Default docker image to be used when the docker runtime is
selected.
</description>
</property>
<property>
<name>yarn.nodemanager.runtime.linux.docker.allowed-container-networks</name>
<value>host,none,bridge</value>