diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index c11e340981..0d8c02f0f9 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -29,6 +29,8 @@ Trunk (Unreleased) HADOOP-11554. Expose HadoopKerberosName as a hadoop subcommand (aw) + HADOOP-11565. Add --slaves shell option (aw) + IMPROVEMENTS HADOOP-8017. Configure hadoop-main pom to get rid of M2E plugin execution diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop b/hadoop-common-project/hadoop-common/src/main/bin/hadoop index 42e4e83b36..6003927047 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop @@ -179,6 +179,11 @@ esac hadoop_verify_user "${COMMAND}" +if [[ ${HADOOP_SLAVE_MODE} = true ]]; then + hadoop_common_slave_mode_execute "${HADOOP_HDFS_HOME}/bin/hdfs" "${HADOOP_USER_PARAMS[@]}" + exit $? +fi + # Always respect HADOOP_OPTS and HADOOP_CLIENT_OPTS hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS" HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}" diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh index f0cabbe3ae..58b871ef63 100644 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh @@ -1,4 +1,4 @@ - +#!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with @@ -91,7 +91,7 @@ hadoop_bootstrap # save these off in case our caller needs them # shellcheck disable=SC2034 -HADOOP_USER_PARAMS="$@" +HADOOP_USER_PARAMS=("$@") HADOOP_DAEMON_MODE="default" @@ -152,6 +152,11 @@ while [[ -z "${_hadoop_common_done}" ]]; do HADOOP_LOGLEVEL="$1" shift ;; + --slaves) + shift + # shellcheck disable=SC2034 + HADOOP_SLAVE_MODE=true + ;; *) _hadoop_common_done=true ;; diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh index 20d7e4a26b..9e4e6b00fa 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh @@ -56,5 +56,17 @@ else hdfsscript="${HADOOP_HDFS_HOME}/bin/hdfs" fi -hadoop_connect_to_hosts "$hdfsscript" \ - --config "${HADOOP_CONF_DIR}" --daemon "${daemonmode}" "$@" +hadoop_error "WARNING: Use of this script to ${daemonmode} HDFS daemons is deprecated." +hadoop_error "WARNING: Attempting to execute replacement \"hdfs --slaves --daemon ${daemonmode}\" instead." + +# +# Original input was usually: +# hadoop-daemons.sh (shell options) (start|stop) (datanode|...) (daemon options) +# we're going to turn this into +# hdfs --slaves --daemon (start|stop) (rest of options) +# +argv=(${HADOOP_USER_PARAMS[@]/start}) +argv=(${argv[@]/stop}) +argv=(${argv[@]/status}) + +${hdfsscript} --slaves --daemon "${daemonmode}" "${argv[@]}" diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh index 142ac22409..cec6b2cc15 100644 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh @@ -405,6 +405,20 @@ function hadoop_connect_to_hosts fi } +function hadoop_common_slave_mode_execute +{ + # + # input should be the command line as given by the user + # in the form of an array + # + local argv=("$@") + + # if --slaves is still on the command line, remove it + # to prevent loops + argv=(${argv[@]/--slaves}) + hadoop_connect_to_hosts -- "${argv[@]}" +} + function hadoop_validate_classname { local class=$1 diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs index 5f8ff1843b..830ca36941 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs @@ -247,6 +247,11 @@ esac hadoop_verify_user "${COMMAND}" +if [[ ${HADOOP_SLAVE_MODE} = true ]]; then + hadoop_common_slave_mode_execute "${HADOOP_HDFS_HOME}/bin/hdfs" "${HADOOP_USER_PARAMS[@]}" + exit $? +fi + if [[ -n "${secure_service}" ]]; then HADOOP_SECURE_USER="${secure_user}" hadoop_verify_secure_prereq diff --git a/hadoop-mapreduce-project/bin/mapred b/hadoop-mapreduce-project/bin/mapred index 6d0c781fc9..d199128956 100755 --- a/hadoop-mapreduce-project/bin/mapred +++ b/hadoop-mapreduce-project/bin/mapred @@ -135,6 +135,11 @@ esac hadoop_verify_user "${COMMAND}" +if [[ ${HADOOP_SLAVE_MODE} = true ]]; then + hadoop_common_slave_mode_execute "${HADOOP_MAPRED_HOME}/bin/mapred" "${HADOOP_USER_PARAMS[@]}" + exit $? +fi + daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out" daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}.pid" diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index b3fe1f3c2e..f1a06a6d3e 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -186,6 +186,11 @@ esac hadoop_verify_user "${COMMAND}" +if [[ ${HADOOP_SLAVE_MODE} = true ]]; then + hadoop_common_slave_mode_execute "${HADOOP_YARN_HOME}/bin/yarn" "${HADOOP_USER_PARAMS[@]}" + exit $? +fi + daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out" daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}.pid" diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh b/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh index 17d3db6412..c6963d9883 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh @@ -41,8 +41,23 @@ else exit 1 fi +yarnscript="${HADOOP_YARN_HOME}/bin/yarn" + daemonmode=$1 shift -hadoop_connect_to_hosts "${HADOOP_YARN_HOME}/bin/yarn" \ - --config "${HADOOP_CONF_DIR}" --daemon "${daemonmode}" "$@" +hadoop_error "WARNING: Use of this script to ${daemonmode} YARN daemons is deprecated." +hadoop_error "WARNING: Attempting to execute replacement \"yarn --slaves --daemon ${daemonmode}\" instead." + +# +# Original input was usually: +# yarn-daemons.sh (shell options) (start|stop) nodemanager (daemon options) +# we're going to turn this into +# yarn --slaves --daemon (start|stop) (rest of options) +# +argv=(${HADOOP_USER_PARAMS[@]/start}) +argv=(${argv[@]/stop}) +argv=(${argv[@]/status}) + +${yarnscript} --slaves --daemon "${daemonmode}" "${argv[@]}" +