From 2dcfc1876e4d73cf85a6b1b7de694b1b4cc54494 Mon Sep 17 00:00:00 2001 From: Miklos Szegedi Date: Fri, 12 Jan 2018 15:03:29 -0800 Subject: [PATCH] YARN-7705. Create the container log directory with correct sticky bit in C code. Contributed by Yufei Gu. --- .../nodemanager/LinuxContainerExecutor.java | 1 + .../localizer/ContainerLocalizer.java | 27 --------- .../impl/container-executor.c | 58 +++++++++++++++++++ .../impl/container-executor.h | 3 + .../native/container-executor/impl/main.c | 6 +- .../test/test-container-executor.c | 11 +++- .../TestLinuxContainerExecutorWithMocks.java | 40 +++++++------ 7 files changed, 98 insertions(+), 48 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java index fc5a04dc60..d359f3159b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java @@ -365,6 +365,7 @@ public void startLocalizer(LocalizerStartContext ctx) PrivilegedOperation.RunAsUserCommand.INITIALIZE_CONTAINER .getValue()), appId, + locId, nmPrivateContainerTokensPath.toUri().getPath().toString(), StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR, localDirs), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java index 541911a0b0..639a69d318 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java @@ -63,7 +63,6 @@ import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.concurrent.HadoopExecutors; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; @@ -447,7 +446,6 @@ public static void main(String[] argv) throws Throwable { // MKDIR $x/$user/appcache/$appid/filecache // LOAD $x/$user/appcache/$appid/appTokens try { - createLogDir(); String user = argv[0]; String appId = argv[1]; String locId = argv[2]; @@ -483,31 +481,6 @@ public static void main(String[] argv) throws Throwable { } } - /** - * Create the log directory, if the directory exists, make sure its permission - * is 750. - */ - private static void createLogDir() { - FileContext localFs; - try { - localFs = FileContext.getLocalFSFileContext(new Configuration()); - - String logDir = System.getProperty( - YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR); - - if (logDir != null && !logDir.trim().isEmpty()) { - Path containerLogPath = new Path(logDir); - FsPermission containerLogDirPerm= new FsPermission((short)0750); - localFs.mkdir(containerLogPath, containerLogDirPerm, true); - // set permission again to make sure the permission is correct - // in case the directory is already there. - localFs.setPermission(containerLogPath, containerLogDirPerm); - } - } catch (IOException e) { - throw new YarnRuntimeException("Unable to create the log dir", e); - } - } - private static void initDirs(Configuration conf, String user, String appId, FileContext lfs, List localDirs) throws IOException { if (null == localDirs || 0 == localDirs.size()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c index c1977e5122..64052fd4eb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c @@ -1094,11 +1094,62 @@ int create_log_dirs(const char *app_id, char * const * log_dirs) { return 0; } +char* get_container_log_directory(const char *log_root, const char* app_id, + const char *container_id) { + return concatenate("%s/%s/%s", "container log dir", 3, log_root, app_id, + container_id); +} + +int create_container_log_dirs(const char *container_id, const char *app_id, + char * const * log_dirs) { + char* const* log_root; + int created_any_dir = 0; + for(log_root=log_dirs; *log_root != NULL; ++log_root) { + char *container_log_dir = get_container_log_directory(*log_root, app_id, + container_id); + + if (container_log_dir == NULL) { + fprintf(LOGFILE, + "Failed to get container log directory name! Log root directory: %s, App id: %s, Container id: %s\n", + *log_root, app_id, container_id); + continue; + } + + int result = check_nm_local_dir(nm_uid, *log_root); + if (result != 0 && container_log_dir != NULL) { + fprintf(LOGFILE, "Unsupported container log directory path (%s) detected.\n", + container_log_dir); + free(container_log_dir); + container_log_dir = NULL; + continue; + } + + if (create_directory_for_user(container_log_dir) != 0) { + fprintf(LOGFILE, "Failed to create container log directory (%s)!\n", + container_log_dir); + free(container_log_dir); + return -1; + } + + if (!created_any_dir) { + created_any_dir = 1; + } + + free(container_log_dir); + } + + if (!created_any_dir) { + fprintf(LOGFILE, "Did not create any container log directory.\n"); + return -1; + } + return 0; +} /** * Function to prepare the application directories for the container. */ int initialize_app(const char *user, const char *app_id, + const char *container_id, const char* nmPrivate_credentials_file, char* const* local_dirs, char* const* log_roots, char* const* args) { @@ -1119,6 +1170,13 @@ int initialize_app(const char *user, const char *app_id, return log_create_result; } + // create the log directories for the container on all disks + int container_log_create_result = create_container_log_dirs(container_id, + app_id, log_roots); + if (container_log_create_result != 0) { + return container_log_create_result; + } + // open up the credentials file int cred_file = open_file_as_nm(nmPrivate_credentials_file); if (cred_file == -1) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h index c5f3d99b7a..5835214811 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h @@ -91,6 +91,7 @@ void free_executor_configurations(); // initialize the application directory int initialize_app(const char *user, const char *app_id, + const char *container_id, const char *credentials, char* const* local_dirs, char* const* log_dirs, char* const* args); @@ -195,6 +196,8 @@ char *get_container_credentials_file(const char* work_dir); */ char* get_app_log_directory(const char* log_root, const char* appid); +char* get_container_log_directory(const char *log_root, const char* app_id, + const char *container_id); /** * Ensure that the given path and all of the parent directories are created * with the desired permissions. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c index ca84d4036a..259719a879 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c @@ -360,13 +360,14 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation) char * resources_value = NULL; switch (command) { case INITIALIZE_CONTAINER: - if (argc < 9) { - fprintf(ERRORFILE, "Too few arguments (%d vs 9) for initialize container\n", + if (argc < 10) { + fprintf(ERRORFILE, "Too few arguments (%d vs 10) for initialize container\n", argc); fflush(ERRORFILE); return INVALID_ARGUMENT_NUMBER; } cmd_input.app_id = argv[optind++]; + cmd_input.container_id = argv[optind++]; cmd_input.cred_file = argv[optind++]; cmd_input.local_dirs = argv[optind++];// good local dirs as a comma separated list cmd_input.log_dirs = argv[optind++];// good log dirs as a comma separated list @@ -563,6 +564,7 @@ int main(int argc, char **argv) { exit_code = initialize_app(cmd_input.yarn_user_name, cmd_input.app_id, + cmd_input.container_id, cmd_input.cred_file, split(cmd_input.local_dirs), split(cmd_input.log_dirs), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c index 1a6d4e6733..8b0c0fabec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c @@ -798,7 +798,8 @@ void test_init_app() { exit(1); } else if (child == 0) { char *final_pgm[] = {"touch", "my-touch-file", 0}; - if (initialize_app(yarn_username, "app_4", TEST_ROOT "/creds.txt", + if (initialize_app(yarn_username, "app_4", "container_1", + TEST_ROOT "/creds.txt", local_dirs, log_dirs, final_pgm) != 0) { printf("FAIL: failed in child\n"); exit(42); @@ -839,6 +840,14 @@ void test_init_app() { exit(1); } free(app_dir); + + char *container_dir = get_container_log_directory(TEST_ROOT "/logs/userlogs", + "app_4", "container_1"); + if (container_dir != NULL && access(container_dir, R_OK) != 0) { + printf("FAIL: failed to create container log directory %s\n", container_dir); + exit(1); + } + free(container_dir); } void test_run_container() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java index 9a5665658a..0632482b92 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java @@ -253,8 +253,6 @@ public void testLaunchCommandWithoutPriority() throws IOException { @Test (timeout = 5000) public void testStartLocalizer() throws IOException { - - InetSocketAddress address = InetSocketAddress.createUnresolved("localhost", 8040); Path nmPrivateCTokensPath= new Path("file:///bin/nmPrivateCTokensPath"); @@ -269,25 +267,31 @@ public void testStartLocalizer() throws IOException { .build()); List result=readMockParams(); - Assert.assertEquals(result.size(), 24); + Assert.assertEquals(result.size(), 25); Assert.assertEquals(result.get(0), YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER); Assert.assertEquals(result.get(1), "test"); Assert.assertEquals(result.get(2), "0" ); - Assert.assertEquals(result.get(3),"application_0" ); - Assert.assertEquals(result.get(4), "/bin/nmPrivateCTokensPath"); - Assert.assertEquals(result.get(8), "-classpath" ); - Assert.assertEquals(result.get(11), "-Xmx256m" ); - Assert.assertEquals(result.get(12), "-Dlog4j.configuration=container-log4j.properties" ); - Assert.assertEquals(result.get(13), "-Dyarn.app.container.log.dir=${yarn.log.dir}/userlogs/application_0/12345"); - Assert.assertEquals(result.get(14), "-Dyarn.app.container.log.filesize=0"); - Assert.assertEquals(result.get(15), "-Dhadoop.root.logger=INFO,CLA"); - Assert.assertEquals(result.get(16), "-Dhadoop.root.logfile=container-localizer-syslog"); - Assert.assertEquals(result.get(17),"org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer" ); - Assert.assertEquals(result.get(18), "test"); - Assert.assertEquals(result.get(19), "application_0"); - Assert.assertEquals(result.get(20),"12345" ); - Assert.assertEquals(result.get(21),"localhost" ); - Assert.assertEquals(result.get(22),"8040" ); + Assert.assertEquals(result.get(3), "application_0"); + Assert.assertEquals(result.get(4), "12345"); + Assert.assertEquals(result.get(5), "/bin/nmPrivateCTokensPath"); + Assert.assertEquals(result.get(9), "-classpath" ); + Assert.assertEquals(result.get(12), "-Xmx256m" ); + Assert.assertEquals(result.get(13), + "-Dlog4j.configuration=container-log4j.properties" ); + Assert.assertEquals(result.get(14), + "-Dyarn.app.container.log.dir=${yarn.log.dir}/userlogs/application_0/12345"); + Assert.assertEquals(result.get(15), + "-Dyarn.app.container.log.filesize=0"); + Assert.assertEquals(result.get(16), "-Dhadoop.root.logger=INFO,CLA"); + Assert.assertEquals(result.get(17), + "-Dhadoop.root.logfile=container-localizer-syslog"); + Assert.assertEquals(result.get(18), + "org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer"); + Assert.assertEquals(result.get(19), "test"); + Assert.assertEquals(result.get(20), "application_0"); + Assert.assertEquals(result.get(21), "12345"); + Assert.assertEquals(result.get(22), "localhost"); + Assert.assertEquals(result.get(23), "8040"); } catch (InterruptedException e) { LOG.error("Error:"+e.getMessage(),e);