From 6176d2b35c85715aae93526236c29540f71ecac8 Mon Sep 17 00:00:00 2001 From: Eric Yang Date: Tue, 22 May 2018 13:44:58 -0400 Subject: [PATCH] YARN-7960. Added security flag no-new-privileges for YARN Docker integration. Contributed by Eric Badger --- .../hadoop-yarn/conf/container-executor.cfg | 1 + .../impl/utils/docker-util.c | 12 +++ .../test/utils/test_docker_util.cc | 90 +++++++++++++++++++ .../src/site/markdown/DockerContainers.md | 1 + 4 files changed, 104 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg index 36676b0530..d19874f242 100644 --- a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg +++ b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg @@ -15,6 +15,7 @@ feature.tc.enabled=false # docker.allowed.rw-mounts=## comma seperate volumes that can be mounted as read-write, add the yarn local and log dirs to this list to run Hadoop jobs # docker.privileged-containers.enabled=false # docker.allowed.volume-drivers=## comma seperated list of allowed volume-drivers +# docker.no-new-privileges.enabled=## enable/disable the no-new-privileges flag for docker run. Set to "true" to enable, disabled by default # The configs below deal with settings for FPGA resource #[fpga] diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c index 099e5b564c..d34a5b21f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c @@ -1374,6 +1374,18 @@ int get_docker_run_command(const char *command_file, const struct configuration reset_args(args); return BUFFER_TOO_SMALL; } + char *no_new_privileges_enabled = + get_configuration_value("docker.no-new-privileges.enabled", + CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf); + if (no_new_privileges_enabled != NULL && + strcasecmp(no_new_privileges_enabled, "True") == 0) { + ret = add_to_args(args, "--security-opt=no-new-privileges"); + if (ret != 0) { + reset_args(args); + return BUFFER_TOO_SMALL; + } + } + free(no_new_privileges_enabled); } free(privileged); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc index e18bf63fb4..613755cec6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc @@ -1545,4 +1545,94 @@ namespace ContainerExecutor { run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_volume_command); } + + TEST_F(TestDockerUtil, test_docker_no_new_privileges) { + + std::string container_executor_contents[] = {"[docker]\n" + " docker.privileged-containers.registries=hadoop\n" + " docker.privileged-containers.enabled=false\n" + " docker.no-new-privileges.enabled=true", + "[docker]\n" + " docker.privileged-containers.registries=hadoop\n" + " docker.privileged-containers.enabled=true\n" + " docker.no-new-privileges.enabled=true", + "[docker]\n" + " docker.privileged-containers.registries=hadoop\n" + " docker.privileged-containers.enabled=true\n" + " docker.no-new-privileges.enabled=true", + "[docker]\n" + " docker.privileged-containers.registries=hadoop\n" + " docker.privileged-containers.enabled=false\n" + " docker.no-new-privileges.enabled=false", + "[docker]\n" + " docker.privileged-containers.registries=hadoop\n" + " docker.privileged-containers.enabled=true\n" + " docker.no-new-privileges.enabled=false"}; + for (int i = 0; i < 2; ++i) { + write_file(container_executor_cfg_file, container_executor_contents[i]); + int ret = read_config(container_executor_cfg_file.c_str(), &container_executor_cfg); + if (ret != 0) { + FAIL(); + } + ret = create_ce_file(); + if (ret != 0) { + std::cerr << "Could not create ce file, skipping test" << std::endl; + return; + } + + std::vector > file_cmd_vec; + file_cmd_vec.push_back(std::make_pair( + "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n" + "image=hadoop/docker-image\n user=nobody", + "run --name=container_e1_12312_11111_02_000001 --user=nobody --security-opt=no-new-privileges " + "--cap-drop=ALL hadoop/docker-image")); + + std::vector > bad_file_cmd_vec; + run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_run_command); + } + + for (int i = 2; i < 3; ++i) { + write_file(container_executor_cfg_file, container_executor_contents[i]); + int ret = read_config(container_executor_cfg_file.c_str(), &container_executor_cfg); + if (ret != 0) { + FAIL(); + } + ret = create_ce_file(); + if (ret != 0) { + std::cerr << "Could not create ce file, skipping test" << std::endl; + return; + } + + std::vector > file_cmd_vec; + file_cmd_vec.push_back(std::make_pair( + "[docker-command-execution]\n docker-command=run\n privileged=true\n" + "name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=root", + "run --name=container_e1_12312_11111_02_000001 --privileged --cap-drop=ALL hadoop/docker-image")); + + std::vector > bad_file_cmd_vec; + run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_run_command); + } + + for (int i = 3; i < 5; ++i) { + write_file(container_executor_cfg_file, container_executor_contents[i]); + int ret = read_config(container_executor_cfg_file.c_str(), &container_executor_cfg); + if (ret != 0) { + FAIL(); + } + ret = create_ce_file(); + if (ret != 0) { + std::cerr << "Could not create ce file, skipping test" << std::endl; + return; + } + + std::vector > file_cmd_vec; + file_cmd_vec.push_back(std::make_pair( + "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n" + "image=hadoop/docker-image\n user=nobody", + "run --name=container_e1_12312_11111_02_000001 --user=nobody --cap-drop=ALL hadoop/docker-image")); + + std::vector > bad_file_cmd_vec; + run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_run_command); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md index 3c392913ec..0f49a06a6d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md @@ -208,6 +208,7 @@ are allowed. It contains the following properties: | `docker.privileged-containers.enabled` | Set to "true" or "false" to enable or disable launching privileged containers. Default value is "false". | | `docker.privileged-containers.registries` | Comma separated list of trusted docker registries for running trusted privileged docker containers. By default, no registries are defined. | | `docker.inspect.max.retries` | Integer value to check docker container readiness. Each inspection is set with 3 seconds delay. Default value of 10 will wait 30 seconds for docker container to become ready before marked as container failed. | +| `docker.no-new-privileges.enabled` | Enable/disable the no-new-privileges flag for docker run. Set to "true" to enable, disabled by default. | Please note that if you wish to run Docker containers that require access to the YARN local directories, you must add them to the docker.allowed.rw-mounts list.