From 0eb4b513b76bc944c31b15cd6558901ae44bf931 Mon Sep 17 00:00:00 2001 From: Allen Wittenauer Date: Thu, 4 Aug 2016 18:08:23 -0700 Subject: [PATCH] HADOOP-13673. Update scripts to be smarter when running with privilege Signed-off-by: Andrew Wang Signed-off-by: Ravi Prakash --- .../hadoop-common/src/main/bin/hadoop | 13 +- .../src/main/bin/hadoop-functions.sh | 215 +++++++++++++++++- .../hadoop-common/src/main/bin/start-all.sh | 23 +- .../hadoop-common/src/main/bin/stop-all.sh | 19 +- .../src/site/markdown/UnixShellGuide.md | 9 + .../scripts/hadoop-functions_test_helper.bash | 6 +- .../src/test/scripts/hadoop_abs.bats | 65 ++++++ .../scripts/hadoop_get_verify_uservar.bats | 21 ++ .../test/scripts/hadoop_privilege_check.bats | 26 +++ .../hadoop-hdfs/src/main/bin/hdfs | 11 + .../hadoop-hdfs/src/main/bin/start-dfs.sh | 50 ++-- .../src/main/bin/start-secure-dns.sh | 23 +- .../hadoop-hdfs/src/main/bin/stop-dfs.sh | 34 ++- .../src/main/bin/stop-secure-dns.sh | 22 +- hadoop-mapreduce-project/bin/mapred | 10 + .../hadoop-yarn/bin/start-yarn.sh | 25 +- .../hadoop-yarn/bin/stop-yarn.sh | 17 +- hadoop-yarn-project/hadoop-yarn/bin/yarn | 11 + 18 files changed, 504 insertions(+), 96 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/test/scripts/hadoop_abs.bats create mode 100644 hadoop-common-project/hadoop-common/src/test/scripts/hadoop_get_verify_uservar.bats create mode 100644 hadoop-common-project/hadoop-common/src/test/scripts/hadoop_privilege_check.bats mode change 100644 => 100755 hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-secure-dns.sh mode change 100644 => 100755 hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-secure-dns.sh diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop b/hadoop-common-project/hadoop-common/src/main/bin/hadoop index bc28c67206..145e348e26 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop @@ -183,13 +183,24 @@ else exit 1 fi -if [ $# = 0 ]; then +# now that we have support code, let's abs MYNAME so we can use it later +MYNAME=$(hadoop_abs "${MYNAME}") + +if [[ $# = 0 ]]; then hadoop_exit_with_usage 1 fi HADOOP_SUBCMD=$1 shift +if hadoop_need_reexec hadoop "${HADOOP_SUBCMD}"; then + hadoop_uservar_su hadoop "${HADOOP_SUBCMD}" \ + "${MYNAME}" \ + "--reexec" \ + "${HADOOP_USER_PARAMS[@]}" + exit $? +fi + hadoop_verify_user "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}" HADOOP_SUBCMD_ARGS=("$@") diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh index 3151023441..b0160b74b3 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh @@ -41,6 +41,42 @@ function hadoop_debug fi } +## @description Given a filename or dir, return the absolute version of it +## @description This works as an alternative to readlink, which isn't +## @description portable. +## @audience public +## @stability stable +## @param fsobj +## @replaceable no +## @return 0 success +## @return 1 failure +## @return stdout abspath +function hadoop_abs +{ + declare obj=$1 + declare dir + declare fn + declare dirret + + if [[ ! -e ${obj} ]]; then + return 1 + elif [[ -d ${obj} ]]; then + dir=${obj} + else + dir=$(dirname -- "${obj}") + fn=$(basename -- "${obj}") + fn="/${fn}" + fi + + dir=$(cd -P -- "${dir}" >/dev/null 2>/dev/null && pwd -P) + dirret=$? + if [[ ${dirret} = 0 ]]; then + echo "${dir}${fn}" + return 0 + fi + return 1 +} + ## @description Given variable $1 delete $2 from it ## @audience public ## @stability stable @@ -79,6 +115,101 @@ function hadoop_verify_entry [[ ${!1} =~ \ ${2}\ ]] } +## @description Check if we are running with privilege +## @description by default, this implementation looks for +## @description EUID=0. For OSes that have true privilege +## @description separation, this should be something more complex +## @audience private +## @stability evolving +## @replaceable yes +## @return 1 = no priv +## @return 0 = priv +function hadoop_privilege_check +{ + [[ "${EUID}" = 0 ]] +} + +## @description Execute a command via su when running as root +## @description if the given user is found or exit with +## @description failure if not. +## @description otherwise just run it. (This is intended to +## @description be used by the start-*/stop-* scripts.) +## @audience private +## @stability evolving +## @replaceable yes +## @param user +## @param commandstring +## @return exitstatus +function hadoop_su +{ + declare user=$1 + shift + declare idret + + if hadoop_privilege_check; then + id -u "${user}" >/dev/null 2>&1 + idret=$? + if [[ ${idret} != 0 ]]; then + hadoop_error "ERROR: Refusing to run as root: ${user} account is not found. Aborting." + return 1 + else + su -l "${user}" -- "$@" + fi + else + "$@" + fi +} + +## @description Execute a command via su when running as root +## @description with extra support for commands that might +## @description legitimately start as root (e.g., datanode) +## @description (This is intended to +## @description be used by the start-*/stop-* scripts.) +## @audience private +## @stability evolving +## @replaceable no +## @param user +## @param commandstring +## @return exitstatus +function hadoop_uservar_su +{ + + ## startup matrix: + # + # if $EUID != 0, then exec + # if $EUID =0 then + # if hdfs_subcmd_user is defined, call hadoop_su to exec + # if hdfs_subcmd_user is not defined, error + # + # For secure daemons, this means both the secure and insecure env vars need to be + # defined. e.g., HDFS_DATANODE_USER=root HDFS_DATANODE_SECURE_USER=hdfs + # This function will pick up the "normal" var, switch to that user, then + # execute the command which will then pick up the "secure" version. + # + + declare program=$1 + declare command=$2 + shift 2 + + declare uprogram + declare ucommand + declare uvar + + if hadoop_privilege_check; then + uvar=$(hadoop_get_verify_uservar "${program}" "${command}") + + if [[ -n "${!uvar}" ]]; then + hadoop_su "${!uvar}" "$@" + else + hadoop_error "ERROR: Attempting to launch ${program} ${command} as root" + hadoop_error "ERROR: but there is no ${uvar} defined. Aborting launch." + return 1 + fi + else + "$@" + fi +} + ## @description Add a subcommand to the usage output ## @audience private ## @stability evolving @@ -343,6 +474,9 @@ function hadoop_bootstrap # daemonization HADOOP_SUBCMD_SUPPORTDAEMONIZATION=false + # by default, we have not been self-re-execed + HADOOP_REEXECED_CMD=false + # shellcheck disable=SC2034 HADOOP_SUBCMD_SECURESERVICE=false @@ -624,9 +758,10 @@ function hadoop_basic_init fi # if for some reason the shell doesn't have $USER defined - # let's define it as 'hadoop' + # (e.g., ssh'd in to execute a command) + # let's get the effective username and use that + USER=${USER:-$(id -nu)} HADOOP_IDENT_STRING=${HADOOP_IDENT_STRING:-$USER} - HADOOP_IDENT_STRING=${HADOOP_IDENT_STRING:-hadoop} HADOOP_LOG_DIR=${HADOOP_LOG_DIR:-"${HADOOP_HOME}/logs"} HADOOP_LOGFILE=${HADOOP_LOGFILE:-hadoop.log} HADOOP_LOGLEVEL=${HADOOP_LOGLEVEL:-INFO} @@ -1400,8 +1535,7 @@ function hadoop_verify_secure_prereq # and you are using pfexec, you'll probably want to change # this. - # ${EUID} comes from the shell itself! - if [[ "${EUID}" -ne 0 ]] && [[ -z "${HADOOP_SECURE_COMMAND}" ]]; then + if ! hadoop_privilege_check && [[ -z "${HADOOP_SECURE_COMMAND}" ]]; then hadoop_error "ERROR: You must be a privileged user in order to run a secure service." exit 1 else @@ -1994,20 +2128,18 @@ function hadoop_secure_daemon_handler esac } -## @description Verify that ${USER} is allowed to execute the -## @description given subcommand. +## @description Get the environment variable used to validate users ## @audience public ## @stability stable ## @replaceable yes ## @param subcommand -## @return will exit on failure conditions -function hadoop_verify_user +## @return string +function hadoop_get_verify_uservar { declare program=$1 declare command=$2 declare uprogram declare ucommand - declare uvar if [[ -z "${BASH_VERSINFO[0]}" ]] \ || [[ "${BASH_VERSINFO[0]}" -lt 4 ]]; then @@ -2018,7 +2150,25 @@ function hadoop_verify_user ucommand=${command^^} fi - uvar="${uprogram}_${ucommand}_USER" + echo "${uprogram}_${ucommand}_USER" +} + +## @description Verify that ${USER} is allowed to execute the +## @description given subcommand. +## @audience public +## @stability stable +## @replaceable yes +## @param command +## @param subcommand +## @return return 0 on success +## @return exit 1 on failure +function hadoop_verify_user +{ + declare program=$1 + declare command=$2 + declare uvar + + uvar=$(hadoop_get_verify_uservar "${program}" "${command}") if [[ -n ${!uvar} ]]; then if [[ ${!uvar} != "${USER}" ]]; then @@ -2026,6 +2176,42 @@ function hadoop_verify_user exit 1 fi fi + return 0 +} + +## @description Verify that ${USER} is allowed to execute the +## @description given subcommand. +## @audience public +## @stability stable +## @replaceable yes +## @param subcommand +## @return 1 on no re-exec needed +## @return 0 on need to re-exec +function hadoop_need_reexec +{ + declare program=$1 + declare command=$2 + declare uvar + + # we've already been re-execed, bail + + if [[ "${HADOOP_REEXECED_CMD}" = true ]]; then + return 1 + fi + + # if we have privilege, and the _USER is defined, and _USER is + # set to someone who isn't us, then yes, we should re-exec. + # otherwise no, don't re-exec and let the system deal with it. + + if hadoop_privilege_check; then + uvar=$(hadoop_get_verify_uservar "${program}" "${command}") + if [[ -n ${!uvar} ]]; then + if [[ ${!uvar} != "${USER}" ]]; then + return 0 + fi + fi + fi + return 1 } ## @description Add custom (program)_(command)_OPTS to HADOOP_OPTS. @@ -2228,6 +2414,15 @@ function hadoop_parse_args shift ((HADOOP_PARSE_COUNTER=HADOOP_PARSE_COUNTER+2)) ;; + --reexec) + shift + if [[ "${HADOOP_REEXECED_CMD}" = true ]]; then + hadoop_error "ERROR: re-exec fork bomb prevention: --reexec already called" + exit 1 + fi + HADOOP_REEXECED_CMD=true + ((HADOOP_PARSE_COUNTER=HADOOP_PARSE_COUNTER+1)) + ;; --workers) shift # shellcheck disable=SC2034 diff --git a/hadoop-common-project/hadoop-common/src/main/bin/start-all.sh b/hadoop-common-project/hadoop-common/src/main/bin/start-all.sh index 142064209d..591622ccd4 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/start-all.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/start-all.sh @@ -15,10 +15,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -echo "This script is deprecated. Use start-dfs.sh and start-yarn.sh instead." -exit 1 - - +## @description catch the ctrl-c +## @audience private +## @stability evolving +## @replaceable no +function hadoop_abort_startall() +{ + exit 1 +} # let's locate libexec... if [[ -n "${HADOOP_HOME}" ]]; then @@ -38,6 +42,16 @@ else echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh." 2>&1 exit 1 fi + +if ! hadoop_privilege_check; then + trap hadoop_abort_startall INT + hadoop_error "WARNING: Attempting to start all Apache Hadoop daemons as ${USER} in 10 seconds." + hadoop_error "WARNING: This is not a recommended production deployment configuration." + hadoop_error "WARNING: Use CTRL-C to abort." + sleep 10 + trap - INT +fi + # start hdfs daemons if hdfs is present if [[ -f "${HADOOP_HDFS_HOME}/sbin/start-dfs.sh" ]]; then "${HADOOP_HDFS_HOME}/sbin/start-dfs.sh" --config "${HADOOP_CONF_DIR}" @@ -49,4 +63,3 @@ if [[ -f "${HADOOP_YARN_HOME}/sbin/start-yarn.sh" ]]; then fi - diff --git a/hadoop-common-project/hadoop-common/src/main/bin/stop-all.sh b/hadoop-common-project/hadoop-common/src/main/bin/stop-all.sh index ee1f6eb569..74e07a3a2e 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/stop-all.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/stop-all.sh @@ -15,12 +15,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +## @description catch the ctrl-c +## @audience private +## @stability evolving +## @replaceable no +function hadoop_abort_stopall() +{ + exit 1 +} # Stop all hadoop daemons. Run this on master node. -echo "This script is deprecated. Use stop-dfs.sh and stop-yarn.sh instead." -exit 1 - # let's locate libexec... if [[ -n "${HADOOP_HOME}" ]]; then HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" @@ -40,6 +45,14 @@ else exit 1 fi +if ! hadoop_privilege_check; then + trap hadoop_abort_stopall INT + hadoop_error "WARNING: Stopping all Apache Hadoop daemons as ${USER} in 10 seconds." + hadoop_error "WARNING: Use CTRL-C to abort." + sleep 10 + trap - INT +fi + # stop hdfs daemons if hdfs is present if [[ -f "${HADOOP_HDFS_HOME}/sbin/stop-dfs.sh" ]]; then "${HADOOP_HDFS_HOME}/sbin/stop-dfs.sh" --config "${HADOOP_CONF_DIR}" diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md b/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md index b130f0f8e7..098f2742ec 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md @@ -103,6 +103,15 @@ In addition, daemons that run in an extra security mode also support `(command)_ Apache Hadoop provides a way to do a user check per-subcommand. While this method is easily circumvented and should not be considered a security-feature, it does provide a mechanism by which to prevent accidents. For example, setting `HDFS_NAMENODE_USER=hdfs` will make the `hdfs namenode` and `hdfs --daemon start namenode` commands verify that the user running the commands are the hdfs user by checking the `USER` environment variable. This also works for non-daemons. Setting `HADOOP_DISTCP_USER=jane` will verify that `USER` is set to `jane` before being allowed to execute the `hadoop distcp` command. +If a \_USER environment variable exists and commands are run with a privilege (e.g., as root; see hadoop_privilege_check in the API documentation), execution will switch to the specified user. For commands that support user account switching for security and therefore have a SECURE\_USER variable, the base \_USER variable needs to be the user that is expected to be used to switch to the SECURE\_USER account. For example: + +```bash +HDFS_DATANODE_USER=root +HDFS_DATANODE_SECURE_USER=hdfs +``` + +Be aware that if the \-\-workers flag is used, the user switch happens *after* ssh is invoked. The multi-daemon start and stop commands in sbin will, however, switch (if appropriate) prior and will therefore use the keys of the specified \_USER. + ## Developer and Advanced Administrator Environment ### Shell Profiles diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop-functions_test_helper.bash b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop-functions_test_helper.bash index cc37268f93..86608edd93 100755 --- a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop-functions_test_helper.bash +++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop-functions_test_helper.bash @@ -16,9 +16,9 @@ setup() { - TMP="${BATS_TEST_DIRNAME}/../../../target/test-dir/bats.$$.${RANDOM}" - mkdir -p ${TMP} - TMP=$(cd -P -- "${TMP}" >/dev/null && pwd -P) + RELTMP="${BATS_TEST_DIRNAME}/../../../target/test-dir/bats.$$.${RANDOM}" + mkdir -p ${RELTMP} + TMP=$(cd -P -- "${RELTMP}" >/dev/null && pwd -P) export TMP TESTBINDIR=$(cd -P -- "$(pwd)" >/dev/null && pwd -P) HADOOP_LIBEXEC_DIR=${TESTBINDIR}/../../main/bin diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_abs.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_abs.bats new file mode 100644 index 0000000000..268bc6875b --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_abs.bats @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load hadoop-functions_test_helper + +create_fake () { + mkdir ${TMP}/j + touch ${TMP}/j/k + ln -s j ${TMP}/l +} + + +@test "hadoop_abs (simple not exist)" { + run hadoop_abs fake + [ "${status}" -eq 1 ] +} + +@test "hadoop_abs (simple dir)" { + create_fake + run hadoop_abs "${TMP}/j" + [ "${output}" = "${TMP}/j" ] +} + +@test "hadoop_abs (simple file)" { + create_fake + run hadoop_abs "${TMP}/j/k" + [ "${output}" = "${TMP}/j/k" ] +} + +@test "hadoop_abs (relative file1)" { + create_fake + run hadoop_abs "${TMP}/j/../j/k" + [ "${output}" = "${TMP}/j/k" ] +} + +@test "hadoop_abs (relative file2)" { + create_fake + run hadoop_abs "${RELTMP}/j/../j/k" + [ "${output}" = "${TMP}/j/k" ] +} + +@test "hadoop_abs (relative dir)" { + create_fake + fred=$(cd -P -- ".." >/dev/null && pwd -P) + run hadoop_abs ".." + [ "${output}" = "${fred}" ] +} + +@test "hadoop_abs (symlink)" { + create_fake + run hadoop_abs "${TMP}/l" + [ "${output}" = "${TMP}/j" ] +} diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_get_verify_uservar.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_get_verify_uservar.bats new file mode 100644 index 0000000000..091fd30f96 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_get_verify_uservar.bats @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load hadoop-functions_test_helper + +@test "hadoop_get_verify_uservar" { + run hadoop_get_verify_uservar cool program + [ "${output}" = "COOL_PROGRAM_USER" ] +} diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_privilege_check.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_privilege_check.bats new file mode 100644 index 0000000000..27916bd280 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_privilege_check.bats @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load hadoop-functions_test_helper + +@test "hadoop_privilege_check " { + if [ ${EUID} = 0 ]; then + result=0 + else + result=1 + fi + run hadoop_privilege_check + [ "${status}" = "${result}" ] +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs index 2299980d39..f095e9b249 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs @@ -239,6 +239,9 @@ else exit 1 fi +# now that we have support code, let's abs MYNAME so we can use it later +MYNAME=$(hadoop_abs "${MYNAME}") + if [[ $# = 0 ]]; then hadoop_exit_with_usage 1 fi @@ -246,6 +249,14 @@ fi HADOOP_SUBCMD=$1 shift +if hadoop_need_reexec hdfs "${HADOOP_SUBCMD}"; then + hadoop_uservar_su hdfs "${HADOOP_SUBCMD}" \ + "${MYNAME}" \ + "--reexec" \ + "${HADOOP_USER_PARAMS[@]}" + exit $? +fi + hadoop_verify_user "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}" HADOOP_SUBCMD_ARGS=("$@") diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh index fc46740d16..9b6a61d420 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh @@ -20,6 +20,21 @@ # Optinally upgrade or rollback dfs state. # Run this on master node. +## startup matrix: +# +# if $EUID != 0, then exec +# if $EUID =0 then +# if hdfs_subcmd_user is defined, su to that user, exec +# if hdfs_subcmd_user is not defined, error +# +# For secure daemons, this means both the secure and insecure env vars need to be +# defined. e.g., HDFS_DATANODE_USER=root HADOOP_SECURE_DN_USER=hdfs +# + +## @description usage info +## @audience private +## @stability evolving +## @replaceable no function hadoop_usage { echo "Usage: start-dfs.sh [-upgrade|-rollback] [-clusterId]" @@ -45,7 +60,6 @@ else exit 1 fi - # get arguments if [[ $# -ge 1 ]]; then startOpt="$1" @@ -77,32 +91,25 @@ if [[ -z "${NAMENODES}" ]]; then fi echo "Starting namenodes on [${NAMENODES}]" - -"${HADOOP_HDFS_HOME}/bin/hdfs" \ +hadoop_uservar_su hdfs namenode "${HADOOP_HDFS_HOME}/bin/hdfs" \ --workers \ --config "${HADOOP_CONF_DIR}" \ --hostnames "${NAMENODES}" \ --daemon start \ namenode ${nameStartOpt} +HADOOP_JUMBO_RETCOUNTER=$? + #--------------------------------------------------------- # datanodes (using default workers file) -if [[ -n "${HADOOP_SECURE_DN_USER}" ]] && - [[ -z "${HADOOP_SECURE_COMMAND}" ]]; then - hadoop_error "ERROR: Attempting to start secure cluster, skipping datanodes. " - hadoop_error "ERROR: Run start-secure-dns.sh as root or configure " - hadoop_error "ERROR: \${HADOOP_SECURE_COMMAND} to complete startup." -else - - echo "Starting datanodes" - - "${HADOOP_HDFS_HOME}/bin/hdfs" \ +echo "Starting datanodes" +hadoop_uservar_su hdfs datanode "${HADOOP_HDFS_HOME}/bin/hdfs" \ --workers \ --config "${HADOOP_CONF_DIR}" \ --daemon start \ datanode ${dataStartOpt} -fi +(( HADOOP_JUMBO_RETCOUNTER=HADOOP_JUMBO_RETCOUNTER + $? )) #--------------------------------------------------------- # secondary namenodes (if any) @@ -113,8 +120,8 @@ if [[ -n "${SECONDARY_NAMENODES}" ]]; then if [[ "${NAMENODES}" =~ , ]]; then - hadoop_error "ERROR: Highly available NameNode is configured." - hadoop_error "ERROR: Skipping SecondaryNameNode." + hadoop_error "WARNING: Highly available NameNode is configured." + hadoop_error "WARNING: Skipping SecondaryNameNode." else @@ -124,12 +131,13 @@ if [[ -n "${SECONDARY_NAMENODES}" ]]; then echo "Starting secondary namenodes [${SECONDARY_NAMENODES}]" - "${HADOOP_HDFS_HOME}/bin/hdfs" \ + hadoop_uservar_su hdfs secondarynamenode "${HADOOP_HDFS_HOME}/bin/hdfs" \ --workers \ --config "${HADOOP_CONF_DIR}" \ --hostnames "${SECONDARY_NAMENODES}" \ --daemon start \ secondarynamenode + (( HADOOP_JUMBO_RETCOUNTER=HADOOP_JUMBO_RETCOUNTER + $? )) fi fi @@ -143,12 +151,13 @@ case "${SHARED_EDITS_DIR}" in JOURNAL_NODES=$(echo "${SHARED_EDITS_DIR}" | sed 's,qjournal://\([^/]*\)/.*,\1,g; s/;/ /g; s/:[0-9]*//g') echo "Starting journal nodes [${JOURNAL_NODES}]" - "${HADOOP_HDFS_HOME}/bin/hdfs" \ + hadoop_uservar_su hdfs journalnode "${HADOOP_HDFS_HOME}/bin/hdfs" \ --workers \ --config "${HADOOP_CONF_DIR}" \ --hostnames "${JOURNAL_NODES}" \ --daemon start \ journalnode + (( HADOOP_JUMBO_RETCOUNTER=HADOOP_JUMBO_RETCOUNTER + $? )) ;; esac @@ -158,12 +167,15 @@ AUTOHA_ENABLED=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey dfs.ha.automati if [[ "${AUTOHA_ENABLED}" = "true" ]]; then echo "Starting ZK Failover Controllers on NN hosts [${NAMENODES}]" - "${HADOOP_HDFS_HOME}/bin/hdfs" \ + hadoop_uservar_su hdfs zkfc "${HADOOP_HDFS_HOME}/bin/hdfs" \ --workers \ --config "${HADOOP_CONF_DIR}" \ --hostnames "${NAMENODES}" \ --daemon start \ zkfc + (( HADOOP_JUMBO_RETCOUNTER=HADOOP_JUMBO_RETCOUNTER + $? )) fi +exit ${HADOOP_JUMBO_RETCOUNTER} + # eof diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-secure-dns.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-secure-dns.sh old mode 100644 new mode 100755 index 7dcbba830e..f3a369c7f4 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-secure-dns.sh +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-secure-dns.sh @@ -17,8 +17,12 @@ # Run as root to start secure datanodes in a security-enabled cluster. - -function hadoop_usage { +## @description usage info +## @audience private +## @stability evolving +## @replaceable no +function hadoop_usage() +{ echo "Usage: start-secure-dns.sh" } @@ -42,12 +46,9 @@ else exit 1 fi -if [[ "${EUID}" -eq 0 ]] && [[ -n "${HADOOP_SECURE_DN_USER}" ]]; then - exec "${HADOOP_HDFS_HOME}/bin/hdfs" \ - --config "${HADOOP_CONF_DIR}" \ - --workers \ - --daemon start \ - datanode -else - echo hadoop_usage_and_exit 1 -fi +echo "Starting datanodes" +hadoop_uservar_su hdfs datanode "${HADOOP_HDFS_HOME}/bin/hdfs" \ + --workers \ + --config "${HADOOP_CONF_DIR}" \ + --daemon start \ + datanode diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh index 797b95b6f7..5d414c8362 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh @@ -19,9 +19,13 @@ # Stop hadoop dfs daemons. # Run this on master node. +## @description usage info +## @audience private +## @stability evolving +## @replaceable no function hadoop_usage { - echo "Usage: stop-dfs.sh [-upgrade|-rollback] [-clusterId]" + echo "Usage: stop-dfs.sh" } this="${BASH_SOURCE-$0}" @@ -55,7 +59,7 @@ fi echo "Stopping namenodes on [${NAMENODES}]" - "${HADOOP_HDFS_HOME}/bin/hdfs" \ + hadoop_uservar_su hdfs namenode "${HADOOP_HDFS_HOME}/bin/hdfs" \ --workers \ --config "${HADOOP_CONF_DIR}" \ --hostnames "${NAMENODES}" \ @@ -65,21 +69,13 @@ echo "Stopping namenodes on [${NAMENODES}]" #--------------------------------------------------------- # datanodes (using default workers file) -if [[ -n "${HADOOP_SECURE_DN_USER}" ]] && -[[ -z "${HADOOP_SECURE_COMMAND}" ]]; then - echo "ERROR: Attempting to stop secure cluster, skipping datanodes. " - echo "Run stop-secure-dns.sh as root or configure " - echo "\${HADOOP_SECURE_COMMAND} to complete stop." -else +echo "Stopping datanodes" - echo "Stopping datanodes" - - "${HADOOP_HDFS_HOME}/bin/hdfs" \ - --workers \ - --config "${HADOOP_CONF_DIR}" \ - --daemon stop \ - datanode -fi +hadoop_uservar_su hdfs datanode "${HADOOP_HDFS_HOME}/bin/hdfs" \ + --workers \ + --config "${HADOOP_CONF_DIR}" \ + --daemon stop \ + datanode #--------------------------------------------------------- # secondary namenodes (if any) @@ -93,7 +89,7 @@ fi if [[ -n "${SECONDARY_NAMENODES}" ]]; then echo "Stopping secondary namenodes [${SECONDARY_NAMENODES}]" - "${HADOOP_HDFS_HOME}/bin/hdfs" \ + hadoop_uservar_su hdfs secondarynamenode "${HADOOP_HDFS_HOME}/bin/hdfs" \ --workers \ --config "${HADOOP_CONF_DIR}" \ --hostnames "${SECONDARY_NAMENODES}" \ @@ -111,7 +107,7 @@ case "${SHARED_EDITS_DIR}" in JOURNAL_NODES=$(echo "${SHARED_EDITS_DIR}" | sed 's,qjournal://\([^/]*\)/.*,\1,g; s/;/ /g; s/:[0-9]*//g') echo "Stopping journal nodes [${JOURNAL_NODES}]" - "${HADOOP_HDFS_HOME}/bin/hdfs" \ + hadoop_uservar_su hdfs journalnode "${HADOOP_HDFS_HOME}/bin/hdfs" \ --workers \ --config "${HADOOP_CONF_DIR}" \ --hostnames "${JOURNAL_NODES}" \ @@ -126,7 +122,7 @@ AUTOHA_ENABLED=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey dfs.ha.automati if [[ "${AUTOHA_ENABLED}" = "true" ]]; then echo "Stopping ZK Failover Controllers on NN hosts [${NAMENODES}]" - "${HADOOP_HDFS_HOME}/bin/hdfs" \ + hadoop_uservar_su hdfs zkfc "${HADOOP_HDFS_HOME}/bin/hdfs" \ --workers \ --config "${HADOOP_CONF_DIR}" \ --hostnames "${NAMENODES}" \ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-secure-dns.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-secure-dns.sh old mode 100644 new mode 100755 index be9683662e..7066ea0421 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-secure-dns.sh +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-secure-dns.sh @@ -17,8 +17,12 @@ # Run as root to stop secure datanodes in a security-enabled cluster. - -function hadoop_usage { +## @description usage info +## @audience private +## @stability evolving +## @replaceable no +function hadoop_usage() +{ echo "Usage: stop-secure-dns.sh" } @@ -42,12 +46,8 @@ else exit 1 fi -if [[ "${EUID}" -eq 0 ]] && [[ -n "${HADOOP_SECURE_DN_USER}" ]]; then - exec "${HADOOP_HDFS_HOME}/bin/hdfs" \ - --config "${HADOOP_CONF_DIR}" \ - --workers \ - --daemon stop \ - datanode -else - echo hadoop_usage_and_exit 1 -fi +hadoop_uservar_su hdfs datanode "${HADOOP_HDFS_HOME}/bin/hdfs" \ + --workers \ + --config "${HADOOP_CONF_DIR}" \ + --daemon stop \ + datanode diff --git a/hadoop-mapreduce-project/bin/mapred b/hadoop-mapreduce-project/bin/mapred index fe7c56ae08..77f0bb81fa 100755 --- a/hadoop-mapreduce-project/bin/mapred +++ b/hadoop-mapreduce-project/bin/mapred @@ -121,6 +121,8 @@ else exit 1 fi +# now that we have support code, let's abs MYNAME so we can use it later +MYNAME=$(hadoop_abs "${MYNAME}") if [ $# = 0 ]; then hadoop_exit_with_usage 1 @@ -129,6 +131,14 @@ fi HADOOP_SUBCMD=$1 shift +if hadoop_need_reexec mapred "${HADOOP_SUBCMD}"; then + hadoop_uservar_su mapred "${HADOOP_SUBCMD}" \ + "${MYNAME}" \ + "--reexec" \ + "${HADOOP_USER_PARAMS[@]}" + exit $? +fi + hadoop_verify_user "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}" HADOOP_SUBCMD_ARGS=("$@") diff --git a/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh b/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh index ecc0140bb3..c009b4381d 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh +++ b/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh @@ -15,14 +15,17 @@ # See the License for the specific language governing permissions and # limitations under the License. - -MYNAME="${BASH_SOURCE-$0}" - +## @description usage info +## @audience private +## @stability evolving +## @replaceable no function hadoop_usage { hadoop_generate_usage "${MYNAME}" false } +MYNAME="${BASH_SOURCE-$0}" + bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P) # let's locate libexec... @@ -42,14 +45,17 @@ else exit 1 fi +HADOOP_JUMBO_RETCOUNTER=0 + # start resourceManager HARM=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey yarn.resourcemanager.ha.enabled 2>&-) if [[ ${HARM} = "false" ]]; then echo "Starting resourcemanager" - "${HADOOP_YARN_HOME}/bin/yarn" \ + hadoop_uservar_su yarn resourcemanager "${HADOOP_YARN_HOME}/bin/yarn" \ --config "${HADOOP_CONF_DIR}" \ --daemon start \ resourcemanager + (( HADOOP_JUMBO_RETCOUNTER=HADOOP_JUMBO_RETCOUNTER + $? )) else logicals=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey yarn.resourcemanager.ha.rm-ids 2>&-) logicals=${logicals//,/ } @@ -59,30 +65,35 @@ else RMHOSTS="${RMHOSTS} ${rmhost}" done echo "Starting resourcemanagers on [${RMHOSTS}]" - "${HADOOP_YARN_HOME}/bin/yarn" \ + hadoop_uservar_su yarn "${HADOOP_YARN_HOME}/bin/yarn" \ --config "${HADOOP_CONF_DIR}" \ --daemon start \ --workers \ --hostnames "${RMHOSTS}" \ resourcemanager + (( HADOOP_JUMBO_RETCOUNTER=HADOOP_JUMBO_RETCOUNTER + $? )) fi # start nodemanager echo "Starting nodemanagers" -"${HADOOP_YARN_HOME}/bin/yarn" \ +hadoop_uservar_su yarn nodemanager "${HADOOP_YARN_HOME}/bin/yarn" \ --config "${HADOOP_CONF_DIR}" \ --workers \ --daemon start \ nodemanager +(( HADOOP_JUMBO_RETCOUNTER=HADOOP_JUMBO_RETCOUNTER + $? )) + # start proxyserver PROXYSERVER=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey yarn.web-proxy.address 2>&- | cut -f1 -d:) if [[ -n ${PROXYSERVER} ]]; then - "${HADOOP_YARN_HOME}/bin/yarn" \ + hadoop_uservar_su yarn proxyserver "${HADOOP_YARN_HOME}/bin/yarn" \ --config "${HADOOP_CONF_DIR}" \ --workers \ --hostnames "${PROXYSERVER}" \ --daemon start \ proxyserver + (( HADOOP_JUMBO_RETCOUNTER=HADOOP_JUMBO_RETCOUNTER + $? )) fi +exit ${HADOOP_JUMBO_RETCOUNTER} diff --git a/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh b/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh index f6c6b781b3..1db6c56645 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh +++ b/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh @@ -15,14 +15,17 @@ # See the License for the specific language governing permissions and # limitations under the License. - -MYNAME="${BASH_SOURCE-$0}" - +## @description usage info +## @audience private +## @stability evolving +## @replaceable no function hadoop_usage { hadoop_generate_usage "${MYNAME}" false } +MYNAME="${BASH_SOURCE-$0}" + bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P) # let's locate libexec... @@ -44,7 +47,7 @@ fi # stop nodemanager echo "Stopping nodemanagers" -"${HADOOP_YARN_HOME}/bin/yarn" \ +hadoop_uservar_su yarn nodemanager "${HADOOP_YARN_HOME}/bin/yarn" \ --config "${HADOOP_CONF_DIR}" \ --workers \ --daemon stop \ @@ -54,7 +57,7 @@ echo "Stopping nodemanagers" HARM=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey yarn.resourcemanager.ha.enabled 2>&-) if [[ ${HARM} = "false" ]]; then echo "Stopping resourcemanager" - "${HADOOP_YARN_HOME}/bin/yarn" \ + hadoop_uservar_su yarn resourcemanager "${HADOOP_YARN_HOME}/bin/yarn" \ --config "${HADOOP_CONF_DIR}" \ --daemon stop \ resourcemanager @@ -67,7 +70,7 @@ else RMHOSTS="${RMHOSTS} ${rmhost}" done echo "Stopping resourcemanagers on [${RMHOSTS}]" - "${HADOOP_YARN_HOME}/bin/yarn" \ + hadoop_uservar_su yarn resourcemanager "${HADOOP_YARN_HOME}/bin/yarn" \ --config "${HADOOP_CONF_DIR}" \ --daemon stop \ --workers \ @@ -79,7 +82,7 @@ fi PROXYSERVER=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey yarn.web-proxy.address 2>&- | cut -f1 -d:) if [[ -n ${PROXYSERVER} ]]; then echo "Stopping proxy server [${PROXYSERVER}]" - "${HADOOP_YARN_HOME}/bin/yarn" \ + hadoop_uservar_su yarn proxyserver "${HADOOP_YARN_HOME}/bin/yarn" \ --config "${HADOOP_CONF_DIR}" \ --workers \ --hostnames "${PROXYSERVER}" \ diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index 804fd1a77f..b64879d93c 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -219,6 +219,9 @@ else exit 1 fi +# now that we have support code, let's abs MYNAME so we can use it later +MYNAME=$(hadoop_abs "${MYNAME}") + # if no args specified, show usage if [[ $# = 0 ]]; then hadoop_exit_with_usage 1 @@ -228,6 +231,14 @@ fi HADOOP_SUBCMD=$1 shift +if hadoop_need_reexec yarn "${HADOOP_SUBCMD}"; then + hadoop_uservar_su yarn "${HADOOP_SUBCMD}" \ + "${MYNAME}" \ + "--reexec" \ + "${HADOOP_USER_PARAMS[@]}" + exit $? +fi + hadoop_verify_user "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}" HADOOP_SUBCMD_ARGS=("$@")