YARN-9379. Can't specify docker runtime through environment. Contributed by caozhiqiang
This commit is contained in:
parent
7fa73fac26
commit
254efc9358
@ -2030,6 +2030,16 @@ public static boolean isAclEnabled(Configuration conf) {
|
|||||||
public static final String DEFAULT_NM_DOCKER_DEFAULT_CONTAINER_NETWORK =
|
public static final String DEFAULT_NM_DOCKER_DEFAULT_CONTAINER_NETWORK =
|
||||||
"host";
|
"host";
|
||||||
|
|
||||||
|
/** The set of runtimes allowed when launching containers using the
|
||||||
|
* DockerContainerRuntime. */
|
||||||
|
public static final String NM_DOCKER_ALLOWED_CONTAINER_RUNTIMES =
|
||||||
|
DOCKER_CONTAINER_RUNTIME_PREFIX + "allowed-container-runtimes";
|
||||||
|
|
||||||
|
/** The set of runtimes allowed when launching containers using the
|
||||||
|
* DockerContainerRuntime. */
|
||||||
|
public static final String[] DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_RUNTIMES =
|
||||||
|
{"runc"};
|
||||||
|
|
||||||
/** Allow host pid namespace for containers. Use with care. */
|
/** Allow host pid namespace for containers. Use with care. */
|
||||||
public static final String NM_DOCKER_ALLOW_HOST_PID_NAMESPACE =
|
public static final String NM_DOCKER_ALLOW_HOST_PID_NAMESPACE =
|
||||||
DOCKER_CONTAINER_RUNTIME_PREFIX + "host-pid-namespace.allowed";
|
DOCKER_CONTAINER_RUNTIME_PREFIX + "host-pid-namespace.allowed";
|
||||||
|
@ -1803,6 +1803,13 @@
|
|||||||
<value>host</value>
|
<value>host</value>
|
||||||
</property>
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<description>The set of runtimes allowed when launching containers using the
|
||||||
|
DockerContainerRuntime.</description>
|
||||||
|
<name>yarn.nodemanager.runtime.linux.docker.allowed-container-runtimes</name>
|
||||||
|
<value>runc</value>
|
||||||
|
</property>
|
||||||
|
|
||||||
<property>
|
<property>
|
||||||
<description>This configuration setting determines whether the host's PID
|
<description>This configuration setting determines whether the host's PID
|
||||||
namespace is allowed for docker containers on this cluster.
|
namespace is allowed for docker containers on this cluster.
|
||||||
|
@ -265,6 +265,9 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
|||||||
@InterfaceAudience.Private
|
@InterfaceAudience.Private
|
||||||
public static final String ENV_DOCKER_CONTAINER_YARN_SYSFS =
|
public static final String ENV_DOCKER_CONTAINER_YARN_SYSFS =
|
||||||
"YARN_CONTAINER_RUNTIME_YARN_SYSFS_ENABLE";
|
"YARN_CONTAINER_RUNTIME_YARN_SYSFS_ENABLE";
|
||||||
|
@InterfaceAudience.Private
|
||||||
|
public static final String ENV_DOCKER_CONTAINER_DOCKER_RUNTIME =
|
||||||
|
"YARN_CONTAINER_RUNTIME_DOCKER_RUNTIME";
|
||||||
public static final String YARN_SYSFS_PATH =
|
public static final String YARN_SYSFS_PATH =
|
||||||
"/hadoop/yarn/sysfs";
|
"/hadoop/yarn/sysfs";
|
||||||
private Configuration conf;
|
private Configuration conf;
|
||||||
@ -275,7 +278,9 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
|
|||||||
private String defaultImageName;
|
private String defaultImageName;
|
||||||
private Boolean defaultImageUpdate;
|
private Boolean defaultImageUpdate;
|
||||||
private Set<String> allowedNetworks = new HashSet<>();
|
private Set<String> allowedNetworks = new HashSet<>();
|
||||||
|
private Set<String> allowedRuntimes = new HashSet<>();
|
||||||
private String defaultNetwork;
|
private String defaultNetwork;
|
||||||
|
private String defaultRuntime;
|
||||||
private CGroupsHandler cGroupsHandler;
|
private CGroupsHandler cGroupsHandler;
|
||||||
private AccessControlList privilegedContainersAcl;
|
private AccessControlList privilegedContainersAcl;
|
||||||
private boolean enableUserReMapping;
|
private boolean enableUserReMapping;
|
||||||
@ -349,6 +354,7 @@ public void initialize(Configuration conf, Context nmContext)
|
|||||||
this.conf = conf;
|
this.conf = conf;
|
||||||
dockerClient = new DockerClient();
|
dockerClient = new DockerClient();
|
||||||
allowedNetworks.clear();
|
allowedNetworks.clear();
|
||||||
|
allowedRuntimes.clear();
|
||||||
defaultROMounts.clear();
|
defaultROMounts.clear();
|
||||||
defaultRWMounts.clear();
|
defaultRWMounts.clear();
|
||||||
defaultTmpfsMounts.clear();
|
defaultTmpfsMounts.clear();
|
||||||
@ -363,6 +369,10 @@ public void initialize(Configuration conf, Context nmContext)
|
|||||||
defaultNetwork = conf.getTrimmed(
|
defaultNetwork = conf.getTrimmed(
|
||||||
YarnConfiguration.NM_DOCKER_DEFAULT_CONTAINER_NETWORK,
|
YarnConfiguration.NM_DOCKER_DEFAULT_CONTAINER_NETWORK,
|
||||||
YarnConfiguration.DEFAULT_NM_DOCKER_DEFAULT_CONTAINER_NETWORK);
|
YarnConfiguration.DEFAULT_NM_DOCKER_DEFAULT_CONTAINER_NETWORK);
|
||||||
|
allowedRuntimes.addAll(Arrays.asList(
|
||||||
|
conf.getTrimmedStrings(
|
||||||
|
YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_RUNTIMES,
|
||||||
|
YarnConfiguration.DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_RUNTIMES)));
|
||||||
|
|
||||||
if(!allowedNetworks.contains(defaultNetwork)) {
|
if(!allowedNetworks.contains(defaultNetwork)) {
|
||||||
String message = "Default network: " + defaultNetwork
|
String message = "Default network: " + defaultNetwork
|
||||||
@ -529,6 +539,19 @@ private void validateContainerNetworkType(String network)
|
|||||||
throw new ContainerExecutionException(msg);
|
throw new ContainerExecutionException(msg);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void validateContainerRuntimeType(String runtime)
|
||||||
|
throws ContainerExecutionException {
|
||||||
|
if (runtime == null || runtime.isEmpty()
|
||||||
|
|| allowedRuntimes.contains(runtime)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
String msg = "Disallowed runtime: '" + runtime
|
||||||
|
+ "' specified. Allowed networks: are " + allowedRuntimes
|
||||||
|
.toString();
|
||||||
|
throw new ContainerExecutionException(msg);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Return whether the YARN container is allowed to run using the host's PID
|
* Return whether the YARN container is allowed to run using the host's PID
|
||||||
* namespace for the Docker container. For this to be allowed, the submitting
|
* namespace for the Docker container. For this to be allowed, the submitting
|
||||||
@ -801,6 +824,7 @@ public void launchContainer(ContainerRuntimeContext ctx)
|
|||||||
String imageName = environment.get(ENV_DOCKER_CONTAINER_IMAGE);
|
String imageName = environment.get(ENV_DOCKER_CONTAINER_IMAGE);
|
||||||
String network = environment.get(ENV_DOCKER_CONTAINER_NETWORK);
|
String network = environment.get(ENV_DOCKER_CONTAINER_NETWORK);
|
||||||
String hostname = environment.get(ENV_DOCKER_CONTAINER_HOSTNAME);
|
String hostname = environment.get(ENV_DOCKER_CONTAINER_HOSTNAME);
|
||||||
|
String runtime = environment.get(ENV_DOCKER_CONTAINER_DOCKER_RUNTIME);
|
||||||
boolean useEntryPoint = checkUseEntryPoint(environment);
|
boolean useEntryPoint = checkUseEntryPoint(environment);
|
||||||
|
|
||||||
if (imageName == null || imageName.isEmpty()) {
|
if (imageName == null || imageName.isEmpty()) {
|
||||||
@ -816,6 +840,8 @@ public void launchContainer(ContainerRuntimeContext ctx)
|
|||||||
|
|
||||||
validateImageName(imageName);
|
validateImageName(imageName);
|
||||||
|
|
||||||
|
validateContainerRuntimeType(runtime);
|
||||||
|
|
||||||
if (defaultImageUpdate) {
|
if (defaultImageUpdate) {
|
||||||
pullImageFromRemote(containerIdStr, imageName);
|
pullImageFromRemote(containerIdStr, imageName);
|
||||||
}
|
}
|
||||||
@ -886,6 +912,9 @@ public void launchContainer(ContainerRuntimeContext ctx)
|
|||||||
}
|
}
|
||||||
|
|
||||||
runCommand.setCapabilities(capabilities);
|
runCommand.setCapabilities(capabilities);
|
||||||
|
if (runtime != null && !runtime.isEmpty()) {
|
||||||
|
runCommand.addRuntime(runtime);
|
||||||
|
}
|
||||||
|
|
||||||
runCommand.addAllReadWriteMountLocations(containerLogDirs);
|
runCommand.addAllReadWriteMountLocations(containerLogDirs);
|
||||||
runCommand.addAllReadWriteMountLocations(applicationLocalDirs);
|
runCommand.addAllReadWriteMountLocations(applicationLocalDirs);
|
||||||
|
@ -2481,6 +2481,56 @@ public void testDockerContainerRelaunch()
|
|||||||
dockerCommands.get(counter));
|
dockerCommands.get(counter));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testLaunchContainersWithSpecificDockerRuntime()
|
||||||
|
throws ContainerExecutionException, PrivilegedOperationException,
|
||||||
|
IOException {
|
||||||
|
DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
|
||||||
|
mockExecutor, mockCGroupsHandler);
|
||||||
|
runtime.initialize(conf, nmContext);
|
||||||
|
|
||||||
|
env.put(DockerLinuxContainerRuntime
|
||||||
|
.ENV_DOCKER_CONTAINER_DOCKER_RUNTIME, "runc");
|
||||||
|
runtime.launchContainer(builder.build());
|
||||||
|
List<String> dockerCommands = readDockerCommands();
|
||||||
|
Assert.assertEquals(14, dockerCommands.size());
|
||||||
|
Assert.assertEquals(" runtime=runc", dockerCommands.get(11));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
public void testContainerLaunchWithAllowedRuntimes()
|
||||||
|
throws ContainerExecutionException, IOException,
|
||||||
|
PrivilegedOperationException {
|
||||||
|
DockerLinuxContainerRuntime runtime =
|
||||||
|
new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler);
|
||||||
|
runtime.initialize(conf, nmContext);
|
||||||
|
|
||||||
|
String disallowedRuntime = "runc2";
|
||||||
|
|
||||||
|
try {
|
||||||
|
env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_DOCKER_RUNTIME,
|
||||||
|
disallowedRuntime);
|
||||||
|
runtime.launchContainer(builder.build());
|
||||||
|
Assert.fail("Runtime was expected to be disallowed: " +
|
||||||
|
disallowedRuntime);
|
||||||
|
} catch (ContainerExecutionException e) {
|
||||||
|
LOG.info("Caught expected exception: " + e);
|
||||||
|
}
|
||||||
|
|
||||||
|
String allowedRuntime = "runc";
|
||||||
|
env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_DOCKER_RUNTIME,
|
||||||
|
allowedRuntime);
|
||||||
|
//this should cause no failures.
|
||||||
|
|
||||||
|
runtime.launchContainer(builder.build());
|
||||||
|
List<String> dockerCommands = readDockerCommands();
|
||||||
|
|
||||||
|
//This is the expected docker invocation for this case
|
||||||
|
Assert.assertEquals(14, dockerCommands.size());
|
||||||
|
Assert.assertEquals(" runtime=runc", dockerCommands.get(11));
|
||||||
|
}
|
||||||
|
|
||||||
private static void verifyStopCommand(List<String> dockerCommands,
|
private static void verifyStopCommand(List<String> dockerCommands,
|
||||||
String signal) {
|
String signal) {
|
||||||
Assert.assertEquals(4, dockerCommands.size());
|
Assert.assertEquals(4, dockerCommands.size());
|
||||||
|
Loading…
Reference in New Issue
Block a user