diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/ContainerShellWebSocket.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/ContainerShellWebSocket.java index 4b7b2acfdd..efcc2ea0ae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/ContainerShellWebSocket.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/ContainerShellWebSocket.java @@ -52,9 +52,16 @@ public class ContainerShellWebSocket { private Session mySession; private Terminal terminal; private LineReader reader; + private boolean sttySet = false; @OnWebSocketMessage public void onText(Session session, String message) throws IOException { + if (!sttySet) { + session.getRemote().sendString("stty -echo"); + session.getRemote().sendString("\r"); + session.getRemote().flush(); + sttySet = true; + } terminal.output().write(message.getBytes(Charset.forName("UTF-8"))); terminal.output().flush(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index a7487f6e1d..28a7f855d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -1121,13 +1121,10 @@ public void shellToContainer(ContainerId containerId, } // Attempt Connect Future fut = client.connect(socket, uri, upgradeRequest); - // Wait for Connect Session session = fut.get(); - // Send a message - session.getRemote().sendString("stty -echo"); - session.getRemote().sendString("\r"); - session.getRemote().flush(); - socket.run(); + if (session.isOpen()) { + socket.run(); + } } finally { client.stop(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c index 65bb8035f3..cba84aca17 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c @@ -143,6 +143,7 @@ int check_executor_permissions(char *executable_file) { fprintf(ERRORFILE, "Error resolving the canonical name for the executable : %s!", strerror(errno)); + fflush(ERRORFILE); return -1; } @@ -151,6 +152,7 @@ int check_executor_permissions(char *executable_file) { if (stat(resolved_path, &filestat) != 0) { fprintf(ERRORFILE, "Could not stat the executable : %s!.\n", strerror(errno)); + fflush(ERRORFILE); return -1; } @@ -161,12 +163,14 @@ int check_executor_permissions(char *executable_file) { if (binary_euid != 0) { fprintf(LOGFILE, "The container-executor binary should be user-owned by root.\n"); + fflush(LOGFILE); return -1; } if (binary_gid != getgid()) { fprintf(LOGFILE, "The configured nodemanager group %d is different from" " the group of the executable %d\n", getgid(), binary_gid); + fflush(LOGFILE); return -1; } @@ -176,12 +180,14 @@ int check_executor_permissions(char *executable_file) { fprintf(LOGFILE, "The container-executor binary should not have write or execute " "for others.\n"); + fflush(LOGFILE); return -1; } // Binary should be setuid executable if ((filestat.st_mode & S_ISUID) == 0) { fprintf(LOGFILE, "The container-executor binary should be set setuid.\n"); + fflush(LOGFILE); return -1; } @@ -201,11 +207,13 @@ static int change_effective_user(uid_t user, gid_t group) { if (setegid(group) != 0) { fprintf(LOGFILE, "Failed to set effective group id %d - %s\n", group, strerror(errno)); + fflush(LOGFILE); return -1; } if (seteuid(user) != 0) { fprintf(LOGFILE, "Failed to set effective user id %d - %s\n", user, strerror(errno)); + fflush(LOGFILE); return -1; } return 0; @@ -230,6 +238,7 @@ static int write_pid_to_cgroup_as_root(const char* cgroup_file, pid_t pid) { if (cgroup_fd == -1) { fprintf(LOGFILE, "Can't open file %s as node manager - %s\n", cgroup_file, strerror(errno)); + fflush(LOGFILE); rc = -1; goto cleanup; } @@ -242,6 +251,7 @@ static int write_pid_to_cgroup_as_root(const char* cgroup_file, pid_t pid) { if (written == -1) { fprintf(LOGFILE, "Failed to write pid to file %s - %s\n", cgroup_file, strerror(errno)); + fflush(LOGFILE); rc = -1; goto cleanup; } @@ -347,6 +357,7 @@ static int write_exit_code_file_as_nm(const char* exit_code_file, if (ecode_fd == -1) { fprintf(LOGFILE, "Can't open file %s - %s\n", tmp_ecode_file, strerror(errno)); + fflush(LOGFILE); rc = -1; goto cleanup; } @@ -358,6 +369,7 @@ static int write_exit_code_file_as_nm(const char* exit_code_file, if (written == -1) { fprintf(LOGFILE, "Failed to write exit code to file %s - %s\n", tmp_ecode_file, strerror(errno)); + fflush(LOGFILE); rc = -1; goto cleanup; } @@ -367,6 +379,7 @@ static int write_exit_code_file_as_nm(const char* exit_code_file, if (rename(tmp_ecode_file, exit_code_file)) { fprintf(LOGFILE, "Can't move exit code file from %s to %s - %s\n", tmp_ecode_file, exit_code_file, strerror(errno)); + fflush(LOGFILE); unlink(tmp_ecode_file); rc = -1; goto cleanup; @@ -396,6 +409,7 @@ static int wait_and_get_exit_code(pid_t pid) { if (waitpid_result < 0) { fprintf(LOGFILE, "error waiting for process %" PRId64 " - %s\n", (int64_t)pid, strerror(errno)); + fflush(LOGFILE); return -1; } @@ -405,6 +419,7 @@ static int wait_and_get_exit_code(pid_t pid) { exit_code = 0x80 + WTERMSIG(child_status); } else { fprintf(LOGFILE, "Unable to determine exit status for pid %" PRId64 "\n", (int64_t)pid); + fflush(LOGFILE); } return exit_code; @@ -440,6 +455,7 @@ int change_user(uid_t user, gid_t group) { fprintf(LOGFILE, "unable to reacquire root - %s\n", strerror(errno)); fprintf(LOGFILE, "Real: %d:%d; Effective: %d:%d\n", getuid(), getgid(), geteuid(), getegid()); + fflush(LOGFILE); return SETUID_OPER_FAILED; } if (setgid(group) != 0) { @@ -447,12 +463,14 @@ int change_user(uid_t user, gid_t group) { strerror(errno)); fprintf(LOGFILE, "Real: %d:%d; Effective: %d:%d\n", getuid(), getgid(), geteuid(), getegid()); + fflush(LOGFILE); return SETUID_OPER_FAILED; } if (setuid(user) != 0) { fprintf(LOGFILE, "unable to set user to %d - %s\n", user, strerror(errno)); fprintf(LOGFILE, "Real: %d:%d; Effective: %d:%d\n", getuid(), getgid(), geteuid(), getegid()); + fflush(LOGFILE); return SETUID_OPER_FAILED; } @@ -533,6 +551,7 @@ char *concatenate(char *concat_pattern, char *return_path_name, if (arg == NULL) { fprintf(LOGFILE, "One of the arguments passed for %s is null.\n", return_path_name); + fflush(LOGFILE); return NULL; } strlen_args += strlen(arg); @@ -545,6 +564,7 @@ char *concatenate(char *concat_pattern, char *return_path_name, return_path = (char *) malloc(str_len); if (return_path == NULL) { fprintf(LOGFILE, "Unable to allocate memory for %s.\n", return_path_name); + fflush(LOGFILE); return NULL; } va_start(ap, numArgs); @@ -594,10 +614,12 @@ int check_nm_local_dir(uid_t caller_uid, const char *nm_root) { int err = stat(nm_root, &info); if (err < 0) { fprintf(LOGFILE, "Error checking file stats for %s %d %s.\n", nm_root, err, strerror(errno)); + fflush(LOGFILE); return 1; } if (caller_uid != info.st_uid) { fprintf(LOGFILE, "Permission mismatch for %s for caller uid: %d, owner uid: %d.\n", nm_root, caller_uid, info.st_uid); + fflush(LOGFILE); return 1; } return 0; @@ -664,6 +686,7 @@ int mkdirs(const char* path, mode_t perm) { npath = strdup(path); if (npath == NULL) { fprintf(LOGFILE, "Not enough memory to copy path string"); + fflush(LOGFILE); return -1; } /* Skip leading slashes. */ @@ -706,6 +729,7 @@ int create_validate_dir(const char* npath, mode_t perm, const char* path, if (errno != EEXIST || stat(npath, &sb) != 0) { fprintf(LOGFILE, "Can't create directory %s - %s\n", npath, strerror(errno)); + fflush(LOGFILE); return -1; } // The directory npath should exist. @@ -726,12 +750,14 @@ int create_validate_dir(const char* npath, mode_t perm, const char* path, int check_dir(const char* npath, mode_t st_mode, mode_t desired, int finalComponent) { if (!S_ISDIR(st_mode)) { fprintf(LOGFILE, "Path %s is file not dir\n", npath); + fflush(LOGFILE); return -1; } else if (finalComponent == 1) { int filePermInt = st_mode & (S_IRWXU | S_IRWXG | S_IRWXO); int desiredInt = desired & (S_IRWXU | S_IRWXG | S_IRWXO); if (filePermInt != desiredInt) { fprintf(LOGFILE, "Path %s has permission %o but needs permission %o.\n", npath, filePermInt, desiredInt); + fflush(LOGFILE); return -1; } } @@ -751,6 +777,7 @@ static int create_container_directories(const char* user, const char *app_id, user_detail == NULL || user_detail->pw_name == NULL) { fprintf(LOGFILE, "Either app_id, container_id or the user passed is null.\n"); + fflush(LOGFILE); return ERROR_CREATE_CONTAINER_DIRECTORIES_ARGUMENTS; } @@ -783,6 +810,7 @@ static int create_container_directories(const char* user, const char *app_id, char *combined_name = malloc(strlen(app_id) + strlen(container_id) + 2); if (combined_name == NULL) { fprintf(LOGFILE, "Malloc of combined name failed\n"); + fflush(LOGFILE); result = OUT_OF_MEMORY; } else { sprintf(combined_name, "%s/%s", app_id, container_id); @@ -795,6 +823,7 @@ static int create_container_directories(const char* user, const char *app_id, } if (strstr(container_log_dir, "..") != 0) { fprintf(LOGFILE, "Unsupported container log directory path detected.\n"); + fflush(LOGFILE); container_log_dir = NULL; } if (container_log_dir == NULL) { @@ -842,6 +871,7 @@ static struct passwd* get_user_info(const char* user) { struct passwd* buffer = malloc(sizeof(struct passwd) + string_size); if (NULL == buffer) { fprintf(LOGFILE, "Failed malloc in get_user_info"); + fflush(LOGFILE); return NULL; } if (getpwnam_r(user, buffer, ((char*)buffer) + sizeof(struct passwd), @@ -849,6 +879,7 @@ static struct passwd* get_user_info(const char* user) { free(buffer); fprintf(LOGFILE, "Can't get user information %s - %s\n", user, strerror(errno)); + fflush(LOGFILE); return NULL; } return result; @@ -921,6 +952,7 @@ struct passwd* check_user(const char *user) { free_values(banned_users); } fprintf(LOGFILE, "Requested user %s is banned\n", user); + fflush(LOGFILE); return NULL; } } @@ -951,6 +983,7 @@ int set_user(const char *user) { if (initgroups(user, user_detail->pw_gid) != 0) { fprintf(LOGFILE, "Error setting supplementary groups for user %s: %s\n", user, strerror(errno)); + fflush(LOGFILE); return -1; } @@ -972,6 +1005,7 @@ static int change_owner(const char* path, uid_t user, gid_t group) { if (chown(path, user, group) != 0) { fprintf(LOGFILE, "Can't chown %s to %d:%d - %s\n", path, user, group, strerror(errno)); + fflush(LOGFILE); return -1; } return 0; @@ -988,6 +1022,7 @@ static int change_owner(const char* path, uid_t user, gid_t group) { if (chown(path, user, group) != 0) { fprintf(LOGFILE, "Can't chown %s to %d:%d - %s\n", path, user, group, strerror(errno)); + fflush(LOGFILE); return -1; } return change_effective_user(old_user, old_group); @@ -1019,20 +1054,24 @@ int create_directory_for_user(const char* path) { if (change_owner(path, user, nm_gid) != 0) { fprintf(LOGFILE, "Failed to chown %s to %d:%d: %s\n", path, user, nm_gid, strerror(errno)); + fflush(LOGFILE); ret = -1; } else if (chmod(path, permissions) != 0) { fprintf(LOGFILE, "Can't chmod %s to add the sticky bit - %s\n", path, strerror(errno)); + fflush(LOGFILE); ret = -1; } } else { fprintf(LOGFILE, "Failed to create directory %s - %s\n", path, strerror(errno)); + fflush(LOGFILE); ret = -1; } } if (change_effective_user(user, group) != 0) { fprintf(LOGFILE, "Failed to change user to %i - %i\n", user, group); + fflush(LOGFILE); ret = -1; } @@ -1053,6 +1092,7 @@ static int open_file_as_nm(const char* filename) { if (result == -1) { fprintf(LOGFILE, "Can't open file %s as node manager - %s\n", filename, strerror(errno)); + fflush(LOGFILE); } if (change_effective_user(user, group)) { result = -1; @@ -1094,6 +1134,7 @@ static int copy_file(int input, const char* in_filename, if (write_result <= 0) { fprintf(LOGFILE, "Error writing to %s - %s\n", out_filename, strerror(errno)); + fflush(LOGFILE); close(out_fd); free(buffer); return -1; @@ -1107,12 +1148,14 @@ static int copy_file(int input, const char* in_filename, if (len < 0) { fprintf(LOGFILE, "Failed to read file %s - %s\n", in_filename, strerror(errno)); + fflush(LOGFILE); close(out_fd); return -1; } if (close(out_fd) != 0) { fprintf(LOGFILE, "Failed to close file %s - %s\n", out_filename, strerror(errno)); + fflush(LOGFILE); return -1; } close(input); @@ -1131,6 +1174,7 @@ int initialize_user(const char *user, char* const* local_dirs) { user_dir = get_user_directory(*local_dir_ptr, user); if (user_dir == NULL) { fprintf(LOGFILE, "Couldn't get userdir directory for %s.\n", user); + fflush(LOGFILE); failed = 1; break; } @@ -1166,6 +1210,7 @@ int create_log_dirs(const char *app_id, char * const * log_dirs) { if (any_one_app_log_dir == NULL) { fprintf(LOGFILE, "Did not create any app-log directories\n"); + fflush(LOGFILE); return -1; } free(any_one_app_log_dir); @@ -1189,17 +1234,20 @@ char *init_log_path(const char *container_log_dir, const char *logfile) { if (change_owner(tmp_buffer, user_detail->pw_uid, nm_gid) != 0) { fprintf(ERRORFILE, "Failed to chown %s to %d:%d: %s\n", tmp_buffer, user_detail->pw_uid, nm_gid, strerror(errno)); + fflush(ERRORFILE); free(tmp_buffer); tmp_buffer = NULL; } else if (chmod(tmp_buffer, permissions) != 0) { fprintf(ERRORFILE, "Can't chmod %s - %s\n", tmp_buffer, strerror(errno)); + fflush(ERRORFILE); free(tmp_buffer); tmp_buffer = NULL; } } else { fprintf(ERRORFILE, "Failed to create file %s - %s\n", tmp_buffer, strerror(errno)); + fflush(ERRORFILE); free(tmp_buffer); tmp_buffer = NULL; } @@ -1217,7 +1265,8 @@ int create_container_log_dirs(const char *container_id, const char *app_id, if (container_log_dir == NULL) { fprintf(LOGFILE, "Failed to get container log directory name! Log root directory: %s, App id: %s, Container id: %s\n", - *log_root, app_id, container_id); + *log_root, app_id, container_id); + fflush(LOGFILE); continue; } @@ -1225,6 +1274,7 @@ int create_container_log_dirs(const char *container_id, const char *app_id, if (result != 0 && container_log_dir != NULL) { fprintf(LOGFILE, "Unsupported container log directory path (%s) detected.\n", container_log_dir); + fflush(LOGFILE); free(container_log_dir); container_log_dir = NULL; continue; @@ -1233,6 +1283,7 @@ int create_container_log_dirs(const char *container_id, const char *app_id, if (create_directory_for_user(container_log_dir) != 0) { fprintf(LOGFILE, "Failed to create container log directory (%s)!\n", container_log_dir); + fflush(LOGFILE); free(container_log_dir); return -1; } @@ -1246,6 +1297,7 @@ int create_container_log_dirs(const char *container_id, const char *app_id, if (!created_any_dir) { fprintf(LOGFILE, "Did not create any container log directory.\n"); + fflush(LOGFILE); return -1; } return 0; @@ -1261,6 +1313,7 @@ int initialize_app(const char *user, const char *app_id, char* const* args) { if (app_id == NULL || user == NULL || user_detail == NULL || user_detail->pw_name == NULL) { fprintf(LOGFILE, "Either app_id is null or the user passed is null.\n"); + fflush(LOGFILE); return INVALID_ARGUMENT_NUMBER; } @@ -1313,6 +1366,7 @@ int initialize_app(const char *user, const char *app_id, if (primary_app_dir == NULL) { fprintf(LOGFILE, "Did not create any app directories\n"); + fflush(LOGFILE); return -1; } @@ -1343,11 +1397,13 @@ int initialize_app(const char *user, const char *app_id, } if (chdir(primary_app_dir) != 0) { fprintf(LOGFILE, "Failed to chdir to app dir - %s\n", strerror(errno)); + fflush(LOGFILE); return -1; } execvp(args[0], args); fprintf(ERRORFILE, "Failure to exec app initialization process - %s\n", strerror(errno)); + fflush(ERRORFILE); return -1; } @@ -1389,13 +1445,12 @@ int run_docker(const char *command_file) { docker_binary, strerror(errno)); fflush(LOGFILE); fflush(ERRORFILE); - free(docker_binary); - free_values(args); exit_code = DOCKER_RUN_FAILED; } else { - free_values(args); exit_code = 0; } + free(docker_binary); + free_values(args); return exit_code; } @@ -1415,13 +1470,14 @@ int exec_container(const char *command_file) { int ret = read_config(command_file, &command_config); if (ret != 0) { free_configuration(&command_config); + free(docker_binary); return INVALID_COMMAND_FILE; } char *value = get_configuration_value("docker-command", DOCKER_COMMAND_FILE_SECTION, &command_config); if (value != NULL && strcasecmp(value, "exec") == 0) { args = construct_docker_command(command_file); - binary = docker_binary; + binary = strdup(docker_binary); docker = 1; } else { value = get_configuration_value("command", COMMAND_FILE_SECTION, &command_config); @@ -1452,19 +1508,22 @@ int exec_container(const char *command_file) { fdm = posix_openpt(O_RDWR); if (fdm < 0) { fprintf(stderr, "Error %d on posix_openpt()\n", errno); - return DOCKER_EXEC_FAILED; + exit_code = DOCKER_EXEC_FAILED; + goto cleanup; } rc = grantpt(fdm); if (rc != 0) { fprintf(stderr, "Error %d on grantpt()\n", errno); - return DOCKER_EXEC_FAILED; + exit_code = DOCKER_EXEC_FAILED; + goto cleanup; } rc = unlockpt(fdm); if (rc != 0) { fprintf(stderr, "Error %d on unlockpt()\n", errno); - return DOCKER_EXEC_FAILED; + exit_code = DOCKER_EXEC_FAILED; + goto cleanup; } // Open the slave PTY @@ -1602,6 +1661,7 @@ int exec_container(const char *command_file) { if (ret != 0) { fprintf(ERRORFILE, "Couldn't execute the container launch with args %s - %s\n", binary, strerror(errno)); + fflush(ERRORFILE); exit_code = DOCKER_EXEC_FAILED; } else { exit_code = 0; @@ -1614,6 +1674,7 @@ cleanup: free(user); free(workdir); free_values(args); + free_values(env); free_configuration(&command_config); return exit_code; } @@ -1834,12 +1895,14 @@ int create_user_filecache_dirs(const char * user, char* const* local_dirs) { char* filecache_dir = get_user_filecache_directory(*ldir_p, user); if (filecache_dir == NULL) { fprintf(LOGFILE, "Couldn't get user filecache directory for %s.\n", user); + fflush(LOGFILE); rc = INITIALIZE_USER_FAILED; break; } if (0 != mkdir(filecache_dir, permissions) && EEXIST != errno) { fprintf(LOGFILE, "Failed to create directory %s - %s\n", filecache_dir, strerror(errno)); + fflush(LOGFILE); free(filecache_dir); rc = INITIALIZE_USER_FAILED; break; @@ -2082,6 +2145,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id, } fprintf(LOGFILE, "Waiting for docker container to finish.\n"); + fflush(LOGFILE); // wait for pid to finish #ifdef __linux @@ -2124,6 +2188,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id, count++; } fprintf(LOGFILE, "Exit code from docker inspect: %d\n", exit_code); + fflush(LOGFILE); cleanup: @@ -2133,6 +2198,7 @@ cleanup: fflush(ERRORFILE); } fprintf(LOGFILE, "Wrote the exit code %d to %s\n", exit_code, exit_code_file); + fflush(LOGFILE); // Drop root privileges if (change_effective_user(prev_uid, user_gid) != 0) { @@ -2251,6 +2317,7 @@ int launch_container_as_user(const char *user, const char *app_id, } fprintf(LOGFILE, "Launching container...\n"); + fflush(LOGFILE); #if HAVE_FCLOSEALL fcloseall(); @@ -2267,6 +2334,7 @@ int launch_container_as_user(const char *user, const char *app_id, if (execlp(script_file_dest, script_file_dest, NULL) != 0) { fprintf(LOGFILE, "Couldn't execute the container launch file %s - %s", script_file_dest, strerror(errno)); + fflush(LOGFILE); exit_code = UNABLE_TO_EXECUTE_CONTAINER_SCRIPT; goto cleanup; } @@ -2294,6 +2362,7 @@ int signal_container_as_user(const char *user, int pid, int sig) { if (kill(-pid,0) < 0) { fprintf(LOGFILE, "Error signalling not exist process group %d " "with signal %d\n", pid, sig); + fflush(LOGFILE); return INVALID_CONTAINER_PID; } @@ -2309,6 +2378,7 @@ int signal_container_as_user(const char *user, int pid, int sig) { } } fprintf(LOGFILE, "Killing process group %d with %d\n", pid, sig); + fflush(LOGFILE); return 0; } @@ -2322,6 +2392,7 @@ static int rmdir_as_nm(const char* path) { if (ret == 0) { if (rmdir(path) != 0 && errno != ENOENT) { fprintf(LOGFILE, "rmdir of %s failed - %s\n", path, strerror(errno)); + fflush(LOGFILE); ret = -1; } } @@ -2418,6 +2489,7 @@ static int recursive_unlink_helper(int dirfd, const char *name, ret = -ret; fprintf(LOGFILE, "is_dir_helper(%s) failed: %s\n", fullpath, strerror(ret)); + fflush(LOGFILE); goto done; } else if (ret == 0) { // is_dir_helper determined that the path is not a directory. @@ -2425,6 +2497,7 @@ static int recursive_unlink_helper(int dirfd, const char *name, if (ret) { fprintf(LOGFILE, "failed to unlink %s: %s\n", fullpath, strerror(ret)); + fflush(LOGFILE); } goto done; } @@ -2441,6 +2514,7 @@ static int recursive_unlink_helper(int dirfd, const char *name, goto done; } fprintf(LOGFILE, "chmod(%s) failed: %s\n", fullpath, strerror(ret)); + fflush(LOGFILE); goto done; } fd = open_helper(dirfd, name); @@ -2452,6 +2526,7 @@ static int recursive_unlink_helper(int dirfd, const char *name, goto done; } fprintf(LOGFILE, "error opening %s: %s\n", fullpath, strerror(ret)); + fflush(LOGFILE); goto done; } if (fstat(fd, &stat) < 0) { @@ -2461,12 +2536,14 @@ static int recursive_unlink_helper(int dirfd, const char *name, goto done; } fprintf(LOGFILE, "failed to stat %s: %s\n", fullpath, strerror(ret)); + fflush(LOGFILE); goto done; } if (!(S_ISDIR(stat.st_mode))) { ret = unlink_helper(dirfd, name, 0); if (ret) { fprintf(LOGFILE, "failed to unlink %s: %s\n", fullpath, strerror(ret)); + fflush(LOGFILE); goto done; } } else { @@ -2480,6 +2557,7 @@ static int recursive_unlink_helper(int dirfd, const char *name, goto done; } fprintf(LOGFILE, "chmod(%s) failed: %s\n", fullpath, strerror(ret)); + fflush(LOGFILE); goto done; } } @@ -2491,6 +2569,7 @@ static int recursive_unlink_helper(int dirfd, const char *name, goto done; } fprintf(LOGFILE, "fopendir(%s) failed: %s\n", fullpath, strerror(ret)); + fflush(LOGFILE); goto done; } while (1) { @@ -2503,6 +2582,7 @@ static int recursive_unlink_helper(int dirfd, const char *name, ret = errno; if (ret && ret != ENOENT) { fprintf(LOGFILE, "readdir(%s) failed: %s\n", fullpath, strerror(ret)); + fflush(LOGFILE); goto done; } break; @@ -2516,6 +2596,7 @@ static int recursive_unlink_helper(int dirfd, const char *name, if (asprintf(&new_fullpath, "%s/%s", fullpath, de->d_name) < 0) { fprintf(LOGFILE, "Failed to allocate string for %s/%s.\n", fullpath, de->d_name); + fflush(LOGFILE); ret = ENOMEM; goto done; } @@ -2529,6 +2610,7 @@ static int recursive_unlink_helper(int dirfd, const char *name, ret = unlink_helper(dirfd, name, AT_REMOVEDIR); if (ret) { fprintf(LOGFILE, "failed to rmdir %s: %s\n", name, strerror(ret)); + fflush(LOGFILE); goto done; } } @@ -2561,12 +2643,14 @@ static int delete_path(const char *full_path, /* Return an error if the path is null. */ if (full_path == NULL) { fprintf(LOGFILE, "Path is null\n"); + fflush(LOGFILE); return PATH_TO_DELETE_IS_NULL; } ret = recursive_unlink_children(full_path); if (ret != 0) { fprintf(LOGFILE, "Error while deleting %s: %d (%s)\n", full_path, ret, strerror(ret)); + fflush(LOGFILE); return -1; } @@ -2584,6 +2668,7 @@ static int delete_path(const char *full_path, if (ret != ENOENT) { fprintf(LOGFILE, "Couldn't delete directory %s - %s\n", full_path, strerror(ret)); + fflush(LOGFILE); return -1; } } @@ -2619,12 +2704,14 @@ int delete_as_user(const char *user, continue; } else { fprintf(LOGFILE, "Could not stat %s - %s\n", *ptr, strerror(errno)); + fflush(LOGFILE); return -1; } } if (!S_ISDIR(sb.st_mode)) { if (!subDirEmptyStr) { fprintf(LOGFILE, "baseDir \"%s\" is a file and cannot contain subdir \"%s\".\n", *ptr, subdir); + fflush(LOGFILE); return -1; } full_path = strdup(*ptr); @@ -2659,6 +2746,7 @@ int list_as_user(const char *target_dir) { // If directory doesn't exist or can't be accessed, error out fprintf(LOGFILE, "Could not stat %s - %s\n", target_dir, strerror(errno)); + fflush(LOGFILE); ret = -1; } else if (!S_ISDIR(sb.st_mode)) { // If it's not a directory, list it as the only file @@ -2686,11 +2774,13 @@ int list_as_user(const char *target_dir) { if (errno != 0) { fprintf(LOGFILE, "Could not read directory %s - %s\n", target_dir, strerror(errno)); + fflush(LOGFILE); ret = -1; } } else { fprintf(LOGFILE, "Could not open directory %s - %s\n", target_dir, strerror(errno)); + fflush(LOGFILE); ret = -1; } } @@ -2719,6 +2809,7 @@ void chown_dir_contents(const char *dir_path, uid_t uid, gid_t gid) { change_owner(path_tmp, uid, gid); } else { fprintf(LOGFILE, "Ignored %s/%s due to length", dir_path, ep->d_name); + fflush(LOGFILE); } } } @@ -2735,6 +2826,7 @@ int is_empty(char *target_dir) { if (!dir) { fprintf(LOGFILE, "Could not open directory %s - %s\n", target_dir, strerror(errno)); + fflush(LOGFILE); return 0; } while ((entry = readdir(dir)) != NULL) { @@ -2745,6 +2837,7 @@ int is_empty(char *target_dir) { continue; } fprintf(LOGFILE, "Directory is not empty %s\n", target_dir); + fflush(LOGFILE); return 0; } return 1; @@ -2759,6 +2852,7 @@ int is_empty(char *target_dir) { int mount_cgroup(const char *pair, const char *hierarchy) { #ifndef __linux fprintf(LOGFILE, "Failed to mount cgroup controller, not supported\n"); + fflush(LOGFILE); return -1; #else size_t len = strlen(pair); @@ -2769,11 +2863,13 @@ int mount_cgroup(const char *pair, const char *hierarchy) { if (controller == NULL || mount_path == NULL) { fprintf(LOGFILE, "Failed to mount cgroup controller; not enough memory\n"); + fflush(LOGFILE); result = OUT_OF_MEMORY; goto cleanup; } if (hierarchy == NULL || strstr(hierarchy, "..") != NULL) { fprintf(LOGFILE, "Unsupported cgroup hierarhy path detected.\n"); + fflush(LOGFILE); result = INVALID_COMMAND_PROVIDED; goto cleanup; } @@ -2781,11 +2877,13 @@ int mount_cgroup(const char *pair, const char *hierarchy) { get_kv_value(pair, mount_path, len) < 0) { fprintf(LOGFILE, "Failed to mount cgroup controller; invalid option: %s\n", pair); + fflush(LOGFILE); result = -1; } else { if (strstr(mount_path, "..") != NULL) { fprintf(LOGFILE, "Unsupported cgroup mount path detected. %s\n", mount_path); + fflush(LOGFILE); result = INVALID_COMMAND_PROVIDED; goto cleanup; } @@ -2805,6 +2903,7 @@ int mount_cgroup(const char *pair, const char *hierarchy) { if (stat(hier_path, &sb) == 0 && (sb.st_uid != nm_uid || sb.st_gid != nm_gid)) { fprintf(LOGFILE, "cgroup hierarchy %s already owned by another user %d\n", hier_path, sb.st_uid); + fflush(LOGFILE); result = INVALID_COMMAND_PROVIDED; goto cleanup; } @@ -2815,6 +2914,7 @@ int mount_cgroup(const char *pair, const char *hierarchy) { } else { fprintf(LOGFILE, "Failed to mount cgroup controller %s at %s - %s\n", controller, mount_path, strerror(errno)); + fflush(LOGFILE); // if controller is already mounted, don't stop trying to mount others if (errno != EBUSY) { result = -1; @@ -2843,6 +2943,7 @@ static int run_traffic_control(const char *opts[], char *command_file) { //too many args to tc if (i == max_tc_args - 1) { fprintf(LOGFILE, "too many args to tc"); + fflush(LOGFILE); return TRAFFIC_CONTROL_EXECUTION_FAILED; } args[i++] = command_file; @@ -2853,6 +2954,7 @@ static int run_traffic_control(const char *opts[], char *command_file) { int exit_code = wait_and_get_exit_code(child_pid); if (exit_code != 0) { fprintf(LOGFILE, "failed to execute tc command!\n"); + fflush(LOGFILE); return TRAFFIC_CONTROL_EXECUTION_FAILED; } return 0; @@ -2860,6 +2962,7 @@ static int run_traffic_control(const char *opts[], char *command_file) { execv(TC_BIN, (char**)args); //if we reach here, exec failed fprintf(LOGFILE, "failed to execute tc command! error: %s\n", strerror(errno)); + fflush(LOGFILE); return TRAFFIC_CONTROL_EXECUTION_FAILED; } } @@ -2985,6 +3088,7 @@ int clean_docker_cgroups_internal(const char *mount_table, const char* container_id) { #ifndef __linux fprintf(LOGFILE, "Failed to clean cgroups, not supported\n"); + fflush(LOGFILE); return -1; #else const char * cgroup_mount_type = "cgroup"; @@ -2998,17 +3102,20 @@ int clean_docker_cgroups_internal(const char *mount_table, if (!mount_table || mount_table[0] == 0) { fprintf(ERRORFILE, "clean_docker_cgroups: Invalid mount table\n"); + fflush(ERRORFILE); rc = -1; goto cleanup; } if (!yarn_hierarchy || yarn_hierarchy[0] == 0) { fprintf(ERRORFILE, "clean_docker_cgroups: Invalid yarn_hierarchy\n"); + fflush(ERRORFILE); rc = -1; goto cleanup; } if (!validate_container_id(container_id)) { fprintf(ERRORFILE, "clean_docker_cgroups: Invalid container_id: %s\n", (container_id == NULL) ? "null" : container_id); + fflush(ERRORFILE); rc = -1; goto cleanup; } @@ -3016,6 +3123,7 @@ int clean_docker_cgroups_internal(const char *mount_table, if (fp == NULL) { fprintf(ERRORFILE, "clean_docker_cgroups: failed to open %s, error %d: %s\n", mount_table, errno, strerror(errno)); + fflush(ERRORFILE); rc = -1; goto cleanup; } @@ -3029,6 +3137,7 @@ int clean_docker_cgroups_internal(const char *mount_table, ret = sscanf(lineptr, " %ms %ms %*s %*s %*s %*s", &mnt_type, &mnt_dir); if (ret != 2) { fprintf(ERRORFILE, "clean_docker_cgroups: Failed to parse line: %s\n", lineptr); + fflush(ERRORFILE); rc = -1; break; } @@ -3037,6 +3146,7 @@ int clean_docker_cgroups_internal(const char *mount_table, } if ((mnt_dir == NULL) || (mnt_dir[0] == 0)) { fprintf(ERRORFILE, "clean_docker_cgroups: skipping mount entry with invalid mnt_dir\n"); + fflush(ERRORFILE); continue; } @@ -3044,6 +3154,7 @@ int clean_docker_cgroups_internal(const char *mount_table, full_path = make_string("%s/%s/%s", mnt_dir, yarn_hierarchy, container_id); if (full_path == NULL) { fprintf(ERRORFILE, "clean_docker_cgroups: Failed to allocate cgroup path.\n"); + fflush(ERRORFILE); rc = -1; break; } @@ -3052,13 +3163,15 @@ int clean_docker_cgroups_internal(const char *mount_table, if (!verify_path_safety(full_path)) { fprintf(ERRORFILE, "clean_docker_cgroups: skipping invalid path: %s\n", full_path); - continue; + fflush(ERRORFILE); + continue; } ret = rmdir(full_path); if ((ret == -1) && (errno != ENOENT)) { fprintf(ERRORFILE, "clean_docker_cgroups: Failed to rmdir cgroup, %s (error=%s)\n", full_path, strerror(errno)); + fflush(ERRORFILE); rc = -1; continue; } @@ -3066,6 +3179,7 @@ int clean_docker_cgroups_internal(const char *mount_table, if (ferror(fp)) { fprintf(ERRORFILE, "clean_docker_cgroups: Error reading %s, error=%d (%s) \n", mount_table, errno, strerror(errno)); + fflush(ERRORFILE); rc = -1; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c index 8cdb8a3412..af540fd58e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c @@ -122,6 +122,7 @@ static void display_usage(FILE *stream) { "[DISABLED] sync yarn sysfs: %2d app-id nm-local-dirs\n", SYNC_YARN_SYSFS); } + fflush(stream); } /* Sets up log files for normal/error logging */ @@ -230,6 +231,7 @@ static void assert_valid_setup(char *argv0) { static void display_feature_disabled_message(const char* name) { fprintf(ERRORFILE, "Feature disabled: %s\n", name); + fflush(ERRORFILE); } /* Use to store parsed input parmeters for various operations */ @@ -458,6 +460,7 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation) cmd_input.container_id = argv[optind++]; if (!validate_container_id(cmd_input.container_id)) { fprintf(ERRORFILE, "Invalid container id %s\n", cmd_input.container_id); + fflush(ERRORFILE); return INVALID_CONTAINER_ID; } cmd_input.cred_file = argv[optind++]; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c index 279d1c700e..86a5d78232 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c @@ -550,7 +550,7 @@ void test_yarn_sysfs() { exit(1); } free(user_dir); - char *app_dir = make_string("%s/usercache/%s/appcache/%s/%s", *local_dir_ptr, username, app_id); + char *app_dir = make_string("%s/usercache/%s/appcache/%s", *local_dir_ptr, username, app_id); if (mkdirs(app_dir, 0750) != 0) { printf("Can not make app directories: %s\n", app_dir); exit(1); @@ -1154,6 +1154,8 @@ static void test_delete_race_internal() { printf("FAIL: return code from delete_as_user is %d\n", ret); exit(1); } + free(app_dir); + free(container_dir); exit(0); } else { // delete application directory