big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-9129. Ensure flush after printing to log plus additional cleanup. Contributed by Eric Yang

Browse Source
This commit is contained in:
Billie Rinaldi 2018-12-19 16:45:23 -08:00
parent e815fd9c49
commit 2e544dc921
5 changed files with 139 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/ContainerShellWebSocket.java
View File

@ -52,9 +52,16 @@ public class ContainerShellWebSocket {
private Session mySession;
private Terminal terminal;
private LineReader reader;
private boolean sttySet = false;
@OnWebSocketMessage
public void onText(Session session, String message) throws IOException {
if (!sttySet) {
session.getRemote().sendString("stty -echo");
session.getRemote().sendString("\r");
session.getRemote().flush();
sttySet = true;
}
terminal.output().write(message.getBytes(Charset.forName("UTF-8")));
terminal.output().flush();
}

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
View File

@ -1121,13 +1121,10 @@ public void shellToContainer(ContainerId containerId,
}
// Attempt Connect
Future<Session> fut = client.connect(socket, uri, upgradeRequest);
// Wait for Connect
Session session = fut.get();
// Send a message
session.getRemote().sendString("stty -echo");
session.getRemote().sendString("\r");
session.getRemote().flush();
if (session.isOpen()) {
socket.run();
}
} finally {
client.stop();
}

128
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
View File

@ -143,6 +143,7 @@ int check_executor_permissions(char *executable_file) {
fprintf(ERRORFILE,
"Error resolving the canonical name for the executable : %s!",
strerror(errno));
fflush(ERRORFILE);
return -1;
}
@ -151,6 +152,7 @@ int check_executor_permissions(char *executable_file) {
if (stat(resolved_path, &filestat) != 0) {
fprintf(ERRORFILE,
"Could not stat the executable : %s!.\n", strerror(errno));
fflush(ERRORFILE);
return -1;
}
@ -161,12 +163,14 @@ int check_executor_permissions(char *executable_file) {
if (binary_euid != 0) {
fprintf(LOGFILE,
"The container-executor binary should be user-owned by root.\n");
fflush(LOGFILE);
return -1;
}
if (binary_gid != getgid()) {
fprintf(LOGFILE, "The configured nodemanager group %d is different from"
" the group of the executable %d\n", getgid(), binary_gid);
fflush(LOGFILE);
return -1;
}
@ -176,12 +180,14 @@ int check_executor_permissions(char *executable_file) {
fprintf(LOGFILE,
"The container-executor binary should not have write or execute "
"for others.\n");
fflush(LOGFILE);
return -1;
}
// Binary should be setuid executable
if ((filestat.st_mode & S_ISUID) == 0) {
fprintf(LOGFILE, "The container-executor binary should be set setuid.\n");
fflush(LOGFILE);
return -1;
}
@ -201,11 +207,13 @@ static int change_effective_user(uid_t user, gid_t group) {
if (setegid(group) != 0) {
fprintf(LOGFILE, "Failed to set effective group id %d - %s\n", group,
strerror(errno));
fflush(LOGFILE);
return -1;
}
if (seteuid(user) != 0) {
fprintf(LOGFILE, "Failed to set effective user id %d - %s\n", user,
strerror(errno));
fflush(LOGFILE);
return -1;
}
return 0;
@ -230,6 +238,7 @@ static int write_pid_to_cgroup_as_root(const char* cgroup_file, pid_t pid) {
if (cgroup_fd == -1) {
fprintf(LOGFILE, "Can't open file %s as node manager - %s\n", cgroup_file,
strerror(errno));
fflush(LOGFILE);
rc = -1;
goto cleanup;
}
@ -242,6 +251,7 @@ static int write_pid_to_cgroup_as_root(const char* cgroup_file, pid_t pid) {
if (written == -1) {
fprintf(LOGFILE, "Failed to write pid to file %s - %s\n",
cgroup_file, strerror(errno));
fflush(LOGFILE);
rc = -1;
goto cleanup;
}
@ -347,6 +357,7 @@ static int write_exit_code_file_as_nm(const char* exit_code_file,
if (ecode_fd == -1) {
fprintf(LOGFILE, "Can't open file %s - %s\n", tmp_ecode_file,
strerror(errno));
fflush(LOGFILE);
rc = -1;
goto cleanup;
}
@ -358,6 +369,7 @@ static int write_exit_code_file_as_nm(const char* exit_code_file,
if (written == -1) {
fprintf(LOGFILE, "Failed to write exit code to file %s - %s\n",
tmp_ecode_file, strerror(errno));
fflush(LOGFILE);
rc = -1;
goto cleanup;
}
@ -367,6 +379,7 @@ static int write_exit_code_file_as_nm(const char* exit_code_file,
if (rename(tmp_ecode_file, exit_code_file)) {
fprintf(LOGFILE, "Can't move exit code file from %s to %s - %s\n",
tmp_ecode_file, exit_code_file, strerror(errno));
fflush(LOGFILE);
unlink(tmp_ecode_file);
rc = -1;
goto cleanup;
@ -396,6 +409,7 @@ static int wait_and_get_exit_code(pid_t pid) {
if (waitpid_result < 0) {
fprintf(LOGFILE, "error waiting for process %" PRId64 " - %s\n", (int64_t)pid, strerror(errno));
fflush(LOGFILE);
return -1;
}
@ -405,6 +419,7 @@ static int wait_and_get_exit_code(pid_t pid) {
exit_code = 0x80 + WTERMSIG(child_status);
} else {
fprintf(LOGFILE, "Unable to determine exit status for pid %" PRId64 "\n", (int64_t)pid);
fflush(LOGFILE);
}
return exit_code;
@ -440,6 +455,7 @@ int change_user(uid_t user, gid_t group) {
fprintf(LOGFILE, "unable to reacquire root - %s\n", strerror(errno));
fprintf(LOGFILE, "Real: %d:%d; Effective: %d:%d\n",
getuid(), getgid(), geteuid(), getegid());
fflush(LOGFILE);
return SETUID_OPER_FAILED;
}
if (setgid(group) != 0) {
@ -447,12 +463,14 @@ int change_user(uid_t user, gid_t group) {
strerror(errno));
fprintf(LOGFILE, "Real: %d:%d; Effective: %d:%d\n",
getuid(), getgid(), geteuid(), getegid());
fflush(LOGFILE);
return SETUID_OPER_FAILED;
}
if (setuid(user) != 0) {
fprintf(LOGFILE, "unable to set user to %d - %s\n", user, strerror(errno));
fprintf(LOGFILE, "Real: %d:%d; Effective: %d:%d\n",
getuid(), getgid(), geteuid(), getegid());
fflush(LOGFILE);
return SETUID_OPER_FAILED;
}
@ -533,6 +551,7 @@ char *concatenate(char *concat_pattern, char *return_path_name,
if (arg == NULL) {
fprintf(LOGFILE, "One of the arguments passed for %s is null.\n",
return_path_name);
fflush(LOGFILE);
return NULL;
}
strlen_args += strlen(arg);
@ -545,6 +564,7 @@ char *concatenate(char *concat_pattern, char *return_path_name,
return_path = (char *) malloc(str_len);
if (return_path == NULL) {
fprintf(LOGFILE, "Unable to allocate memory for %s.\n", return_path_name);
fflush(LOGFILE);
return NULL;
}
va_start(ap, numArgs);
@ -594,10 +614,12 @@ int check_nm_local_dir(uid_t caller_uid, const char *nm_root) {
int err = stat(nm_root, &info);
if (err < 0) {
fprintf(LOGFILE, "Error checking file stats for %s %d %s.\n", nm_root, err, strerror(errno));
fflush(LOGFILE);
return 1;
}
if (caller_uid != info.st_uid) {
fprintf(LOGFILE, "Permission mismatch for %s for caller uid: %d, owner uid: %d.\n", nm_root, caller_uid, info.st_uid);
fflush(LOGFILE);
return 1;
}
return 0;
@ -664,6 +686,7 @@ int mkdirs(const char* path, mode_t perm) {
npath = strdup(path);
if (npath == NULL) {
fprintf(LOGFILE, "Not enough memory to copy path string");
fflush(LOGFILE);
return -1;
}
/* Skip leading slashes. */
@ -706,6 +729,7 @@ int create_validate_dir(const char* npath, mode_t perm, const char* path,
if (errno != EEXIST || stat(npath, &sb) != 0) {
fprintf(LOGFILE, "Can't create directory %s - %s\n", npath,
strerror(errno));
fflush(LOGFILE);
return -1;
}
// The directory npath should exist.
@ -726,12 +750,14 @@ int create_validate_dir(const char* npath, mode_t perm, const char* path,
int check_dir(const char* npath, mode_t st_mode, mode_t desired, int finalComponent) {
if (!S_ISDIR(st_mode)) {
fprintf(LOGFILE, "Path %s is file not dir\n", npath);
fflush(LOGFILE);
return -1;
} else if (finalComponent == 1) {
int filePermInt = st_mode & (S_IRWXU | S_IRWXG | S_IRWXO);
int desiredInt = desired & (S_IRWXU | S_IRWXG | S_IRWXO);
if (filePermInt != desiredInt) {
fprintf(LOGFILE, "Path %s has permission %o but needs permission %o.\n", npath, filePermInt, desiredInt);
fflush(LOGFILE);
return -1;
}
}
@ -751,6 +777,7 @@ static int create_container_directories(const char* user, const char *app_id,
user_detail == NULL || user_detail->pw_name == NULL) {
fprintf(LOGFILE,
"Either app_id, container_id or the user passed is null.\n");
fflush(LOGFILE);
return ERROR_CREATE_CONTAINER_DIRECTORIES_ARGUMENTS;
}
@ -783,6 +810,7 @@ static int create_container_directories(const char* user, const char *app_id,
char *combined_name = malloc(strlen(app_id) + strlen(container_id) + 2);
if (combined_name == NULL) {
fprintf(LOGFILE, "Malloc of combined name failed\n");
fflush(LOGFILE);
result = OUT_OF_MEMORY;
} else {
sprintf(combined_name, "%s/%s", app_id, container_id);
@ -795,6 +823,7 @@ static int create_container_directories(const char* user, const char *app_id,
}
if (strstr(container_log_dir, "..") != 0) {
fprintf(LOGFILE, "Unsupported container log directory path detected.\n");
fflush(LOGFILE);
container_log_dir = NULL;
}
if (container_log_dir == NULL) {
@ -842,6 +871,7 @@ static struct passwd* get_user_info(const char* user) {
struct passwd* buffer = malloc(sizeof(struct passwd) + string_size);
if (NULL == buffer) {
fprintf(LOGFILE, "Failed malloc in get_user_info");
fflush(LOGFILE);
return NULL;
}
if (getpwnam_r(user, buffer, ((char*)buffer) + sizeof(struct passwd),
@ -849,6 +879,7 @@ static struct passwd* get_user_info(const char* user) {
free(buffer);
fprintf(LOGFILE, "Can't get user information %s - %s\n", user,
strerror(errno));
fflush(LOGFILE);
return NULL;
}
return result;
@ -921,6 +952,7 @@ struct passwd* check_user(const char *user) {
free_values(banned_users);
}
fprintf(LOGFILE, "Requested user %s is banned\n", user);
fflush(LOGFILE);
return NULL;
}
}
@ -951,6 +983,7 @@ int set_user(const char *user) {
if (initgroups(user, user_detail->pw_gid) != 0) {
fprintf(LOGFILE, "Error setting supplementary groups for user %s: %s\n",
user, strerror(errno));
fflush(LOGFILE);
return -1;
}
@ -972,6 +1005,7 @@ static int change_owner(const char* path, uid_t user, gid_t group) {
if (chown(path, user, group) != 0) {
fprintf(LOGFILE, "Can't chown %s to %d:%d - %s\n", path, user, group,
strerror(errno));
fflush(LOGFILE);
return -1;
}
return 0;
@ -988,6 +1022,7 @@ static int change_owner(const char* path, uid_t user, gid_t group) {
if (chown(path, user, group) != 0) {
fprintf(LOGFILE, "Can't chown %s to %d:%d - %s\n", path, user, group,
strerror(errno));
fflush(LOGFILE);
return -1;
}
return change_effective_user(old_user, old_group);
@ -1019,20 +1054,24 @@ int create_directory_for_user(const char* path) {
if (change_owner(path, user, nm_gid) != 0) {
fprintf(LOGFILE, "Failed to chown %s to %d:%d: %s\n", path, user, nm_gid,
strerror(errno));
fflush(LOGFILE);
ret = -1;
} else if (chmod(path, permissions) != 0) {
fprintf(LOGFILE, "Can't chmod %s to add the sticky bit - %s\n",
path, strerror(errno));
fflush(LOGFILE);
ret = -1;
}
} else {
fprintf(LOGFILE, "Failed to create directory %s - %s\n", path,
strerror(errno));
fflush(LOGFILE);
ret = -1;
}
}
if (change_effective_user(user, group) != 0) {
fprintf(LOGFILE, "Failed to change user to %i - %i\n", user, group);
fflush(LOGFILE);
ret = -1;
}
@ -1053,6 +1092,7 @@ static int open_file_as_nm(const char* filename) {
if (result == -1) {
fprintf(LOGFILE, "Can't open file %s as node manager - %s\n", filename,
strerror(errno));
fflush(LOGFILE);
}
if (change_effective_user(user, group)) {
result = -1;
@ -1094,6 +1134,7 @@ static int copy_file(int input, const char* in_filename,
if (write_result <= 0) {
fprintf(LOGFILE, "Error writing to %s - %s\n", out_filename,
strerror(errno));
fflush(LOGFILE);
close(out_fd);
free(buffer);
return -1;
@ -1107,12 +1148,14 @@ static int copy_file(int input, const char* in_filename,
if (len < 0) {
fprintf(LOGFILE, "Failed to read file %s - %s\n", in_filename,
strerror(errno));
fflush(LOGFILE);
close(out_fd);
return -1;
}
if (close(out_fd) != 0) {
fprintf(LOGFILE, "Failed to close file %s - %s\n", out_filename,
strerror(errno));
fflush(LOGFILE);
return -1;
}
close(input);
@ -1131,6 +1174,7 @@ int initialize_user(const char *user, char* const* local_dirs) {
user_dir = get_user_directory(*local_dir_ptr, user);
if (user_dir == NULL) {
fprintf(LOGFILE, "Couldn't get userdir directory for %s.\n", user);
fflush(LOGFILE);
failed = 1;
break;
}
@ -1166,6 +1210,7 @@ int create_log_dirs(const char *app_id, char * const * log_dirs) {
if (any_one_app_log_dir == NULL) {
fprintf(LOGFILE, "Did not create any app-log directories\n");
fflush(LOGFILE);
return -1;
}
free(any_one_app_log_dir);
@ -1189,17 +1234,20 @@ char *init_log_path(const char *container_log_dir, const char *logfile) {
if (change_owner(tmp_buffer, user_detail->pw_uid, nm_gid) != 0) {
fprintf(ERRORFILE, "Failed to chown %s to %d:%d: %s\n", tmp_buffer, user_detail->pw_uid, nm_gid,
strerror(errno));
fflush(ERRORFILE);
free(tmp_buffer);
tmp_buffer = NULL;
} else if (chmod(tmp_buffer, permissions) != 0) {
fprintf(ERRORFILE, "Can't chmod %s - %s\n",
tmp_buffer, strerror(errno));
fflush(ERRORFILE);
free(tmp_buffer);
tmp_buffer = NULL;
}
} else {
fprintf(ERRORFILE, "Failed to create file %s - %s\n", tmp_buffer,
strerror(errno));
fflush(ERRORFILE);
free(tmp_buffer);
tmp_buffer = NULL;
}
@ -1218,6 +1266,7 @@ int create_container_log_dirs(const char *container_id, const char *app_id,
fprintf(LOGFILE,
"Failed to get container log directory name! Log root directory: %s, App id: %s, Container id: %s\n",
*log_root, app_id, container_id);
fflush(LOGFILE);
continue;
}
@ -1225,6 +1274,7 @@ int create_container_log_dirs(const char *container_id, const char *app_id,
if (result != 0 && container_log_dir != NULL) {
fprintf(LOGFILE, "Unsupported container log directory path (%s) detected.\n",
container_log_dir);
fflush(LOGFILE);
free(container_log_dir);
container_log_dir = NULL;
continue;
@ -1233,6 +1283,7 @@ int create_container_log_dirs(const char *container_id, const char *app_id,
if (create_directory_for_user(container_log_dir) != 0) {
fprintf(LOGFILE, "Failed to create container log directory (%s)!\n",
container_log_dir);
fflush(LOGFILE);
free(container_log_dir);
return -1;
}
@ -1246,6 +1297,7 @@ int create_container_log_dirs(const char *container_id, const char *app_id,
if (!created_any_dir) {
fprintf(LOGFILE, "Did not create any container log directory.\n");
fflush(LOGFILE);
return -1;
}
return 0;
@ -1261,6 +1313,7 @@ int initialize_app(const char *user, const char *app_id,
char* const* args) {
if (app_id == NULL || user == NULL || user_detail == NULL || user_detail->pw_name == NULL) {
fprintf(LOGFILE, "Either app_id is null or the user passed is null.\n");
fflush(LOGFILE);
return INVALID_ARGUMENT_NUMBER;
}
@ -1313,6 +1366,7 @@ int initialize_app(const char *user, const char *app_id,
if (primary_app_dir == NULL) {
fprintf(LOGFILE, "Did not create any app directories\n");
fflush(LOGFILE);
return -1;
}
@ -1343,11 +1397,13 @@ int initialize_app(const char *user, const char *app_id,
}
if (chdir(primary_app_dir) != 0) {
fprintf(LOGFILE, "Failed to chdir to app dir - %s\n", strerror(errno));
fflush(LOGFILE);
return -1;
}
execvp(args[0], args);
fprintf(ERRORFILE, "Failure to exec app initialization process - %s\n",
strerror(errno));
fflush(ERRORFILE);
return -1;
}
@ -1389,13 +1445,12 @@ int run_docker(const char *command_file) {
docker_binary, strerror(errno));
fflush(LOGFILE);
fflush(ERRORFILE);
free(docker_binary);
free_values(args);
exit_code = DOCKER_RUN_FAILED;
} else {
free_values(args);
exit_code = 0;
}
free(docker_binary);
free_values(args);
return exit_code;
}
@ -1415,13 +1470,14 @@ int exec_container(const char *command_file) {
int ret = read_config(command_file, &command_config);
if (ret != 0) {
free_configuration(&command_config);
free(docker_binary);
return INVALID_COMMAND_FILE;
}
char *value = get_configuration_value("docker-command", DOCKER_COMMAND_FILE_SECTION, &command_config);
if (value != NULL && strcasecmp(value, "exec") == 0) {
args = construct_docker_command(command_file);
binary = docker_binary;
binary = strdup(docker_binary);
docker = 1;
} else {
value = get_configuration_value("command", COMMAND_FILE_SECTION, &command_config);
@ -1452,19 +1508,22 @@ int exec_container(const char *command_file) {
fdm = posix_openpt(O_RDWR);
if (fdm < 0) {
fprintf(stderr, "Error %d on posix_openpt()\n", errno);
return DOCKER_EXEC_FAILED;
exit_code = DOCKER_EXEC_FAILED;
goto cleanup;
}
rc = grantpt(fdm);
if (rc != 0) {
fprintf(stderr, "Error %d on grantpt()\n", errno);
return DOCKER_EXEC_FAILED;
exit_code = DOCKER_EXEC_FAILED;
goto cleanup;
}
rc = unlockpt(fdm);
if (rc != 0) {
fprintf(stderr, "Error %d on unlockpt()\n", errno);
return DOCKER_EXEC_FAILED;
exit_code = DOCKER_EXEC_FAILED;
goto cleanup;
}
// Open the slave PTY
@ -1602,6 +1661,7 @@ int exec_container(const char *command_file) {
if (ret != 0) {
fprintf(ERRORFILE, "Couldn't execute the container launch with args %s - %s\n",
binary, strerror(errno));
fflush(ERRORFILE);
exit_code = DOCKER_EXEC_FAILED;
} else {
exit_code = 0;
@ -1614,6 +1674,7 @@ cleanup:
free(user);
free(workdir);
free_values(args);
free_values(env);
free_configuration(&command_config);
return exit_code;
}
@ -1834,12 +1895,14 @@ int create_user_filecache_dirs(const char * user, char* const* local_dirs) {
char* filecache_dir = get_user_filecache_directory(*ldir_p, user);
if (filecache_dir == NULL) {
fprintf(LOGFILE, "Couldn't get user filecache directory for %s.\n", user);
fflush(LOGFILE);
rc = INITIALIZE_USER_FAILED;
break;
}
if (0 != mkdir(filecache_dir, permissions) && EEXIST != errno) {
fprintf(LOGFILE, "Failed to create directory %s - %s\n", filecache_dir,
strerror(errno));
fflush(LOGFILE);
free(filecache_dir);
rc = INITIALIZE_USER_FAILED;
break;
@ -2082,6 +2145,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
}
fprintf(LOGFILE, "Waiting for docker container to finish.\n");
fflush(LOGFILE);
// wait for pid to finish
#ifdef __linux
@ -2124,6 +2188,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
count++;
}
fprintf(LOGFILE, "Exit code from docker inspect: %d\n", exit_code);
fflush(LOGFILE);
cleanup:
@ -2133,6 +2198,7 @@ cleanup:
fflush(ERRORFILE);
}
fprintf(LOGFILE, "Wrote the exit code %d to %s\n", exit_code, exit_code_file);
fflush(LOGFILE);
// Drop root privileges
if (change_effective_user(prev_uid, user_gid) != 0) {
@ -2251,6 +2317,7 @@ int launch_container_as_user(const char *user, const char *app_id,
}
fprintf(LOGFILE, "Launching container...\n");
fflush(LOGFILE);
#if HAVE_FCLOSEALL
fcloseall();
@ -2267,6 +2334,7 @@ int launch_container_as_user(const char *user, const char *app_id,
if (execlp(script_file_dest, script_file_dest, NULL) != 0) {
fprintf(LOGFILE, "Couldn't execute the container launch file %s - %s",
script_file_dest, strerror(errno));
fflush(LOGFILE);
exit_code = UNABLE_TO_EXECUTE_CONTAINER_SCRIPT;
goto cleanup;
}
@ -2294,6 +2362,7 @@ int signal_container_as_user(const char *user, int pid, int sig) {
if (kill(-pid,0) < 0) {
fprintf(LOGFILE, "Error signalling not exist process group %d "
"with signal %d\n", pid, sig);
fflush(LOGFILE);
return INVALID_CONTAINER_PID;
}
@ -2309,6 +2378,7 @@ int signal_container_as_user(const char *user, int pid, int sig) {
}
}
fprintf(LOGFILE, "Killing process group %d with %d\n", pid, sig);
fflush(LOGFILE);
return 0;
}
@ -2322,6 +2392,7 @@ static int rmdir_as_nm(const char* path) {
if (ret == 0) {
if (rmdir(path) != 0 && errno != ENOENT) {
fprintf(LOGFILE, "rmdir of %s failed - %s\n", path, strerror(errno));
fflush(LOGFILE);
ret = -1;
}
}
@ -2418,6 +2489,7 @@ static int recursive_unlink_helper(int dirfd, const char *name,
ret = -ret;
fprintf(LOGFILE, "is_dir_helper(%s) failed: %s\n",
fullpath, strerror(ret));
fflush(LOGFILE);
goto done;
} else if (ret == 0) {
// is_dir_helper determined that the path is not a directory.
@ -2425,6 +2497,7 @@ static int recursive_unlink_helper(int dirfd, const char *name,
if (ret) {
fprintf(LOGFILE, "failed to unlink %s: %s\n",
fullpath, strerror(ret));
fflush(LOGFILE);
}
goto done;
}
@ -2441,6 +2514,7 @@ static int recursive_unlink_helper(int dirfd, const char *name,
goto done;
}
fprintf(LOGFILE, "chmod(%s) failed: %s\n", fullpath, strerror(ret));
fflush(LOGFILE);
goto done;
}
fd = open_helper(dirfd, name);
@ -2452,6 +2526,7 @@ static int recursive_unlink_helper(int dirfd, const char *name,
goto done;
}
fprintf(LOGFILE, "error opening %s: %s\n", fullpath, strerror(ret));
fflush(LOGFILE);
goto done;
}
if (fstat(fd, &stat) < 0) {
@ -2461,12 +2536,14 @@ static int recursive_unlink_helper(int dirfd, const char *name,
goto done;
}
fprintf(LOGFILE, "failed to stat %s: %s\n", fullpath, strerror(ret));
fflush(LOGFILE);
goto done;
}
if (!(S_ISDIR(stat.st_mode))) {
ret = unlink_helper(dirfd, name, 0);
if (ret) {
fprintf(LOGFILE, "failed to unlink %s: %s\n", fullpath, strerror(ret));
fflush(LOGFILE);
goto done;
}
} else {
@ -2480,6 +2557,7 @@ static int recursive_unlink_helper(int dirfd, const char *name,
goto done;
}
fprintf(LOGFILE, "chmod(%s) failed: %s\n", fullpath, strerror(ret));
fflush(LOGFILE);
goto done;
}
}
@ -2491,6 +2569,7 @@ static int recursive_unlink_helper(int dirfd, const char *name,
goto done;
}
fprintf(LOGFILE, "fopendir(%s) failed: %s\n", fullpath, strerror(ret));
fflush(LOGFILE);
goto done;
}
while (1) {
@ -2503,6 +2582,7 @@ static int recursive_unlink_helper(int dirfd, const char *name,
ret = errno;
if (ret && ret != ENOENT) {
fprintf(LOGFILE, "readdir(%s) failed: %s\n", fullpath, strerror(ret));
fflush(LOGFILE);
goto done;
}
break;
@ -2516,6 +2596,7 @@ static int recursive_unlink_helper(int dirfd, const char *name,
if (asprintf(&new_fullpath, "%s/%s", fullpath, de->d_name) < 0) {
fprintf(LOGFILE, "Failed to allocate string for %s/%s.\n",
fullpath, de->d_name);
fflush(LOGFILE);
ret = ENOMEM;
goto done;
}
@ -2529,6 +2610,7 @@ static int recursive_unlink_helper(int dirfd, const char *name,
ret = unlink_helper(dirfd, name, AT_REMOVEDIR);
if (ret) {
fprintf(LOGFILE, "failed to rmdir %s: %s\n", name, strerror(ret));
fflush(LOGFILE);
goto done;
}
}
@ -2561,12 +2643,14 @@ static int delete_path(const char *full_path,
/* Return an error if the path is null. */
if (full_path == NULL) {
fprintf(LOGFILE, "Path is null\n");
fflush(LOGFILE);
return PATH_TO_DELETE_IS_NULL;
}
ret = recursive_unlink_children(full_path);
if (ret != 0) {
fprintf(LOGFILE, "Error while deleting %s: %d (%s)\n",
full_path, ret, strerror(ret));
fflush(LOGFILE);
return -1;
}
@ -2584,6 +2668,7 @@ static int delete_path(const char *full_path,
if (ret != ENOENT) {
fprintf(LOGFILE, "Couldn't delete directory %s - %s\n",
full_path, strerror(ret));
fflush(LOGFILE);
return -1;
}
}
@ -2619,12 +2704,14 @@ int delete_as_user(const char *user,
continue;
} else {
fprintf(LOGFILE, "Could not stat %s - %s\n", *ptr, strerror(errno));
fflush(LOGFILE);
return -1;
}
}
if (!S_ISDIR(sb.st_mode)) {
if (!subDirEmptyStr) {
fprintf(LOGFILE, "baseDir \"%s\" is a file and cannot contain subdir \"%s\".\n", *ptr, subdir);
fflush(LOGFILE);
return -1;
}
full_path = strdup(*ptr);
@ -2659,6 +2746,7 @@ int list_as_user(const char *target_dir) {
// If directory doesn't exist or can't be accessed, error out
fprintf(LOGFILE, "Could not stat %s - %s\n", target_dir,
strerror(errno));
fflush(LOGFILE);
ret = -1;
} else if (!S_ISDIR(sb.st_mode)) {
// If it's not a directory, list it as the only file
@ -2686,11 +2774,13 @@ int list_as_user(const char *target_dir) {
if (errno != 0) {
fprintf(LOGFILE, "Could not read directory %s - %s\n", target_dir,
strerror(errno));
fflush(LOGFILE);
ret = -1;
}
} else {
fprintf(LOGFILE, "Could not open directory %s - %s\n", target_dir,
strerror(errno));
fflush(LOGFILE);
ret = -1;
}
}
@ -2719,6 +2809,7 @@ void chown_dir_contents(const char *dir_path, uid_t uid, gid_t gid) {
change_owner(path_tmp, uid, gid);
} else {
fprintf(LOGFILE, "Ignored %s/%s due to length", dir_path, ep->d_name);
fflush(LOGFILE);
}
}
}
@ -2735,6 +2826,7 @@ int is_empty(char *target_dir) {
if (!dir) {
fprintf(LOGFILE, "Could not open directory %s - %s\n", target_dir,
strerror(errno));
fflush(LOGFILE);
return 0;
}
while ((entry = readdir(dir)) != NULL) {
@ -2745,6 +2837,7 @@ int is_empty(char *target_dir) {
continue;
}
fprintf(LOGFILE, "Directory is not empty %s\n", target_dir);
fflush(LOGFILE);
return 0;
}
return 1;
@ -2759,6 +2852,7 @@ int is_empty(char *target_dir) {
int mount_cgroup(const char *pair, const char *hierarchy) {
#ifndef __linux
fprintf(LOGFILE, "Failed to mount cgroup controller, not supported\n");
fflush(LOGFILE);
return -1;
#else
size_t len = strlen(pair);
@ -2769,11 +2863,13 @@ int mount_cgroup(const char *pair, const char *hierarchy) {
if (controller == NULL || mount_path == NULL) {
fprintf(LOGFILE, "Failed to mount cgroup controller; not enough memory\n");
fflush(LOGFILE);
result = OUT_OF_MEMORY;
goto cleanup;
}
if (hierarchy == NULL || strstr(hierarchy, "..") != NULL) {
fprintf(LOGFILE, "Unsupported cgroup hierarhy path detected.\n");
fflush(LOGFILE);
result = INVALID_COMMAND_PROVIDED;
goto cleanup;
}
@ -2781,11 +2877,13 @@ int mount_cgroup(const char *pair, const char *hierarchy) {
get_kv_value(pair, mount_path, len) < 0) {
fprintf(LOGFILE, "Failed to mount cgroup controller; invalid option: %s\n",
pair);
fflush(LOGFILE);
result = -1;
} else {
if (strstr(mount_path, "..") != NULL) {
fprintf(LOGFILE, "Unsupported cgroup mount path detected. %s\n",
mount_path);
fflush(LOGFILE);
result = INVALID_COMMAND_PROVIDED;
goto cleanup;
}
@ -2805,6 +2903,7 @@ int mount_cgroup(const char *pair, const char *hierarchy) {
if (stat(hier_path, &sb) == 0 &&
(sb.st_uid != nm_uid || sb.st_gid != nm_gid)) {
fprintf(LOGFILE, "cgroup hierarchy %s already owned by another user %d\n", hier_path, sb.st_uid);
fflush(LOGFILE);
result = INVALID_COMMAND_PROVIDED;
goto cleanup;
}
@ -2815,6 +2914,7 @@ int mount_cgroup(const char *pair, const char *hierarchy) {
} else {
fprintf(LOGFILE, "Failed to mount cgroup controller %s at %s - %s\n",
controller, mount_path, strerror(errno));
fflush(LOGFILE);
// if controller is already mounted, don't stop trying to mount others
if (errno != EBUSY) {
result = -1;
@ -2843,6 +2943,7 @@ static int run_traffic_control(const char *opts[], char *command_file) {
//too many args to tc
if (i == max_tc_args - 1) {
fprintf(LOGFILE, "too many args to tc");
fflush(LOGFILE);
return TRAFFIC_CONTROL_EXECUTION_FAILED;
}
args[i++] = command_file;
@ -2853,6 +2954,7 @@ static int run_traffic_control(const char *opts[], char *command_file) {
int exit_code = wait_and_get_exit_code(child_pid);
if (exit_code != 0) {
fprintf(LOGFILE, "failed to execute tc command!\n");
fflush(LOGFILE);
return TRAFFIC_CONTROL_EXECUTION_FAILED;
}
return 0;
@ -2860,6 +2962,7 @@ static int run_traffic_control(const char *opts[], char *command_file) {
execv(TC_BIN, (char**)args);
//if we reach here, exec failed
fprintf(LOGFILE, "failed to execute tc command! error: %s\n", strerror(errno));
fflush(LOGFILE);
return TRAFFIC_CONTROL_EXECUTION_FAILED;
}
}
@ -2985,6 +3088,7 @@ int clean_docker_cgroups_internal(const char *mount_table,
const char* container_id) {
#ifndef __linux
fprintf(LOGFILE, "Failed to clean cgroups, not supported\n");
fflush(LOGFILE);
return -1;
#else
const char * cgroup_mount_type = "cgroup";
@ -2998,17 +3102,20 @@ int clean_docker_cgroups_internal(const char *mount_table,
if (!mount_table || mount_table[0] == 0) {
fprintf(ERRORFILE, "clean_docker_cgroups: Invalid mount table\n");
fflush(ERRORFILE);
rc = -1;
goto cleanup;
}
if (!yarn_hierarchy || yarn_hierarchy[0] == 0) {
fprintf(ERRORFILE, "clean_docker_cgroups: Invalid yarn_hierarchy\n");
fflush(ERRORFILE);
rc = -1;
goto cleanup;
}
if (!validate_container_id(container_id)) {
fprintf(ERRORFILE, "clean_docker_cgroups: Invalid container_id: %s\n",
(container_id == NULL) ? "null" : container_id);
fflush(ERRORFILE);
rc = -1;
goto cleanup;
}
@ -3016,6 +3123,7 @@ int clean_docker_cgroups_internal(const char *mount_table,
if (fp == NULL) {
fprintf(ERRORFILE, "clean_docker_cgroups: failed to open %s, error %d: %s\n",
mount_table, errno, strerror(errno));
fflush(ERRORFILE);
rc = -1;
goto cleanup;
}
@ -3029,6 +3137,7 @@ int clean_docker_cgroups_internal(const char *mount_table,
ret = sscanf(lineptr, " %ms %ms %*s %*s %*s %*s", &mnt_type, &mnt_dir);
if (ret != 2) {
fprintf(ERRORFILE, "clean_docker_cgroups: Failed to parse line: %s\n", lineptr);
fflush(ERRORFILE);
rc = -1;
break;
}
@ -3037,6 +3146,7 @@ int clean_docker_cgroups_internal(const char *mount_table,
}
if ((mnt_dir == NULL) || (mnt_dir[0] == 0)) {
fprintf(ERRORFILE, "clean_docker_cgroups: skipping mount entry with invalid mnt_dir\n");
fflush(ERRORFILE);
continue;
}
@ -3044,6 +3154,7 @@ int clean_docker_cgroups_internal(const char *mount_table,
full_path = make_string("%s/%s/%s", mnt_dir, yarn_hierarchy, container_id);
if (full_path == NULL) {
fprintf(ERRORFILE, "clean_docker_cgroups: Failed to allocate cgroup path.\n");
fflush(ERRORFILE);
rc = -1;
break;
}
@ -3052,6 +3163,7 @@ int clean_docker_cgroups_internal(const char *mount_table,
if (!verify_path_safety(full_path)) {
fprintf(ERRORFILE,
"clean_docker_cgroups: skipping invalid path: %s\n", full_path);
fflush(ERRORFILE);
continue;
}
@ -3059,6 +3171,7 @@ int clean_docker_cgroups_internal(const char *mount_table,
if ((ret == -1) && (errno != ENOENT)) {
fprintf(ERRORFILE, "clean_docker_cgroups: Failed to rmdir cgroup, %s (error=%s)\n",
full_path, strerror(errno));
fflush(ERRORFILE);
rc = -1;
continue;
}
@ -3066,6 +3179,7 @@ int clean_docker_cgroups_internal(const char *mount_table,
if (ferror(fp)) {
fprintf(ERRORFILE, "clean_docker_cgroups: Error reading %s, error=%d (%s) \n",
mount_table, errno, strerror(errno));
fflush(ERRORFILE);
rc = -1;
}

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
View File

@ -122,6 +122,7 @@ static void display_usage(FILE *stream) {
"[DISABLED] sync yarn sysfs: %2d app-id nm-local-dirs\n",
SYNC_YARN_SYSFS);
}
fflush(stream);
}
/* Sets up log files for normal/error logging */
@ -230,6 +231,7 @@ static void assert_valid_setup(char *argv0) {
static void display_feature_disabled_message(const char* name) {
fprintf(ERRORFILE, "Feature disabled: %s\n", name);
fflush(ERRORFILE);
}
/* Use to store parsed input parmeters for various operations */
@ -458,6 +460,7 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation)
cmd_input.container_id = argv[optind++];
if (!validate_container_id(cmd_input.container_id)) {
fprintf(ERRORFILE, "Invalid container id %s\n", cmd_input.container_id);
fflush(ERRORFILE);
return INVALID_CONTAINER_ID;
}
cmd_input.cred_file = argv[optind++];

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
View File

@ -550,7 +550,7 @@ void test_yarn_sysfs() {
exit(1);
}
free(user_dir);
char *app_dir = make_string("%s/usercache/%s/appcache/%s/%s", *local_dir_ptr, username, app_id);
char *app_dir = make_string("%s/usercache/%s/appcache/%s", *local_dir_ptr, username, app_id);
if (mkdirs(app_dir, 0750) != 0) {
printf("Can not make app directories: %s\n", app_dir);
exit(1);
@ -1154,6 +1154,8 @@ static void test_delete_race_internal() {
printf("FAIL: return code from delete_as_user is %d\n", ret);
exit(1);
}
free(app_dir);
free(container_dir);
exit(0);
} else {
// delete application directory