big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-8656. container-executor should not write cgroup tasks files for docker containers. Contributed by Jim Brennan

Browse Source
This commit is contained in:
Jason Lowe 2018-08-16 10:06:17 -05:00
parent 6df606f1b4
commit cb21eaa026
5 changed files with 10 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
View File

@ -1156,7 +1156,6 @@ private PrivilegedOperation buildLaunchOp(ContainerRuntimeContext ctx,
List<String> localDirs = ctx.getExecutionAttribute(LOCAL_DIRS);
@SuppressWarnings("unchecked")
List<String> logDirs = ctx.getExecutionAttribute(LOG_DIRS);
String resourcesOpts = ctx.getExecutionAttribute(RESOURCES_OPTIONS);
PrivilegedOperation launchOp = new PrivilegedOperation(
PrivilegedOperation.OperationType.LAUNCH_DOCKER_CONTAINER);
@ -1174,8 +1173,7 @@ private PrivilegedOperation buildLaunchOp(ContainerRuntimeContext ctx,
localDirs),
StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
logDirs),
commandFile,
resourcesOpts);
commandFile);
String tcCommandFile = ctx.getExecutionAttribute(TC_COMMAND_FILE);

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
View File

@ -1547,9 +1547,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
const char *container_id, const char *work_dir,
const char *script_name, const char *cred_file,
const char *pid_file, char* const* local_dirs,
char* const* log_dirs, const char *command_file,
const char *resources_key,
char* const* resources_values) {
char* const* log_dirs, const char *command_file) {
int exit_code = -1;
char *script_file_dest = NULL;
char *cred_file_dest = NULL;
@ -1732,23 +1730,6 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
}
if (pid != 0) {
#ifdef __linux
fprintf(LOGFILE, "Writing to cgroup task files...\n");
// cgroups-based resource enforcement
if (resources_key != NULL && ! strcmp(resources_key, "cgroups")) {
// write pid to cgroups
char* const* cgroup_ptr;
for (cgroup_ptr = resources_values; cgroup_ptr != NULL &&
*cgroup_ptr != NULL; ++cgroup_ptr) {
if (strcmp(*cgroup_ptr, "none") != 0 &&
write_pid_to_cgroup_as_root(*cgroup_ptr, pid) != 0) {
exit_code = WRITE_CGROUP_FAILED;
goto cleanup;
}
}
}
#endif
// write pid to pidfile
fprintf(LOGFILE, "Writing pid file...\n");
if (pid_file == NULL

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
View File

@ -104,8 +104,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
const char *script_name, const char *cred_file,
const char *pid_file, char* const* local_dirs,
char* const* log_dirs,
const char *command_file,const char *resources_key,
char* const* resources_values);
const char *command_file);
/*
* Function used to launch a container as the provided user. It does the following :

26
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
View File

@ -430,8 +430,8 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation)
case LAUNCH_DOCKER_CONTAINER:
if(is_docker_support_enabled()) {
//kill me now.
if (!(argc == 14 || argc == 15)) {
fprintf(ERRORFILE, "Wrong number of arguments (%d vs 14 or 15) for"
if (!(argc == 13 || argc == 14)) {
fprintf(ERRORFILE, "Wrong number of arguments (%d vs 13 or 14) for"
" launch docker container\n", argc);
fflush(ERRORFILE);
return INVALID_ARGUMENT_NUMBER;
@ -448,21 +448,8 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation)
// good log dirs as a comma separated list
cmd_input.log_dirs = argv[optind++];
cmd_input.docker_command_file = argv[optind++];
// key,value pair describing resources
resources = argv[optind++];
resources_key = malloc(strlen(resources));
resources_value = malloc(strlen(resources));
if (get_kv_key(resources, resources_key, strlen(resources)) < 0 ||
get_kv_value(resources, resources_value, strlen(resources)) < 0) {
fprintf(ERRORFILE, "Invalid arguments for cgroups resources: %s",
resources);
fflush(ERRORFILE);
free(resources_key);
free(resources_value);
return INVALID_ARGUMENT_NUMBER;
}
//network isolation through tc
if (argc == 15) {
if (argc == 14) {
if(is_tc_support_enabled()) {
cmd_input.traffic_control_command_file = argv[optind++];
} else {
@ -471,9 +458,6 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation)
}
}
cmd_input.resources_key = resources_key;
cmd_input.resources_value = resources_value;
cmd_input.resources_values = split(resources_value);
*operation = RUN_AS_USER_LAUNCH_DOCKER_CONTAINER;
return 0;
} else {
@ -653,9 +637,7 @@ int main(int argc, char **argv) {
cmd_input.pid_file,
split(cmd_input.local_dirs),
split(cmd_input.log_dirs),
cmd_input.docker_command_file,
cmd_input.resources_key,
cmd_input.resources_values);
cmd_input.docker_command_file);
break;
case RUN_AS_USER_LAUNCH_CONTAINER:
if (cmd_input.traffic_control_command_file != NULL) {

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
View File

@ -355,9 +355,9 @@ private PrivilegedOperation capturePrivilegedOperationAndVerifyArgs()
List<String> args = op.getArguments();
//This invocation of container-executor should use 13 arguments in a
//This invocation of container-executor should use 12 arguments in a
// specific order
int expected = 13;
int expected = 12;
int counter = 1;
Assert.assertEquals(expected, args.size());
Assert.assertEquals(user, args.get(counter++));
@ -373,7 +373,6 @@ private PrivilegedOperation capturePrivilegedOperationAndVerifyArgs()
Assert.assertEquals(pidFilePath.toString(), args.get(counter++));
Assert.assertEquals(localDirs.get(0), args.get(counter++));
Assert.assertEquals(logDirs.get(0), args.get(counter++));
Assert.assertEquals(resourcesOptions, args.get(++counter));
return op;
}
@ -2116,7 +2115,7 @@ public void testLaunchContainerWithDockerTokens()
List<String> args = op.getArguments();
int expectedArgs = 13;
int expectedArgs = 12;
int argsCounter = 0;
Assert.assertEquals(expectedArgs, args.size());
Assert.assertEquals(runAsUser, args.get(argsCounter++));
@ -2134,7 +2133,6 @@ public void testLaunchContainerWithDockerTokens()
Assert.assertEquals(localDirs.get(0), args.get(argsCounter++));
Assert.assertEquals(logDirs.get(0), args.get(argsCounter++));
String dockerCommandFile = args.get(argsCounter++);
Assert.assertEquals(resourcesOptions, args.get(argsCounter));
List<String> dockerCommands = Files
.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));