HADOOP-11565. Add --slaves shell option (aw)

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -29,6 +29,8 @@ Trunk (Unreleased)
 
     HADOOP-11554. Expose HadoopKerberosName as a hadoop subcommand (aw)
 
+    HADOOP-11565. Add --slaves shell option (aw)
+
   IMPROVEMENTS
 
     HADOOP-8017. Configure hadoop-main pom to get rid of M2E plugin execution

hadoop-common-project/hadoop-common/src/main/bin/hadoop

@@ -179,6 +179,11 @@ esac
 
 hadoop_verify_user "${COMMAND}"
 
+if [[ ${HADOOP_SLAVE_MODE} = true ]]; then
+  hadoop_common_slave_mode_execute "${HADOOP_HDFS_HOME}/bin/hdfs" "${HADOOP_USER_PARAMS[@]}"
+  exit $?
+fi
+
 # Always respect HADOOP_OPTS and HADOOP_CLIENT_OPTS
 hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
 HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"

hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh

@@ -1,4 +1,4 @@
-
+#!/usr/bin/env bash
 #
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
@@ -91,7 +91,7 @@ hadoop_bootstrap
 
 # save these off in case our caller needs them
 # shellcheck disable=SC2034
-HADOOP_USER_PARAMS="$@"
+HADOOP_USER_PARAMS=("$@")
 
 HADOOP_DAEMON_MODE="default"
 
@@ -152,6 +152,11 @@ while [[ -z "${_hadoop_common_done}" ]]; do
       HADOOP_LOGLEVEL="$1"
       shift
     ;;
+    --slaves)
+      shift
+      # shellcheck disable=SC2034
+      HADOOP_SLAVE_MODE=true
+    ;;
     *)
       _hadoop_common_done=true
     ;;

hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh

@@ -56,5 +56,17 @@ else
   hdfsscript="${HADOOP_HDFS_HOME}/bin/hdfs"
 fi
 
-hadoop_connect_to_hosts "$hdfsscript" \
-    --config "${HADOOP_CONF_DIR}" --daemon "${daemonmode}" "$@"
+hadoop_error "WARNING: Use of this script to ${daemonmode} HDFS daemons is deprecated."
+hadoop_error "WARNING: Attempting to execute replacement \"hdfs --slaves --daemon ${daemonmode}\" instead."
+
+#
+# Original input was usually:
+#  hadoop-daemons.sh (shell options) (start|stop) (datanode|...) (daemon options)
+# we're going to turn this into
+#  hdfs --slaves --daemon (start|stop) (rest of options)
+#
+argv=(${HADOOP_USER_PARAMS[@]/start})
+argv=(${argv[@]/stop})
+argv=(${argv[@]/status})
+
+${hdfsscript} --slaves --daemon "${daemonmode}" "${argv[@]}"

hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh

@@ -405,6 +405,20 @@ function hadoop_connect_to_hosts
   fi
 }
 
+function hadoop_common_slave_mode_execute
+{
+  #
+  # input should be the command line as given by the user
+  # in the form of an array
+  #
+  local argv=("$@")
+
+  # if --slaves is still on the command line, remove it
+  # to prevent loops
+  argv=(${argv[@]/--slaves})
+  hadoop_connect_to_hosts -- "${argv[@]}"
+}
+
 function hadoop_validate_classname
 {
   local class=$1

hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs

@@ -247,6 +247,11 @@ esac
 
 hadoop_verify_user "${COMMAND}"
 
+if [[ ${HADOOP_SLAVE_MODE} = true ]]; then
+  hadoop_common_slave_mode_execute "${HADOOP_HDFS_HOME}/bin/hdfs" "${HADOOP_USER_PARAMS[@]}"
+  exit $?
+fi
+
 if [[ -n "${secure_service}" ]]; then
   HADOOP_SECURE_USER="${secure_user}"
   hadoop_verify_secure_prereq

hadoop-mapreduce-project/bin/mapred

@@ -135,6 +135,11 @@ esac
 
 hadoop_verify_user "${COMMAND}"
 
+if [[ ${HADOOP_SLAVE_MODE} = true ]]; then
+  hadoop_common_slave_mode_execute "${HADOOP_MAPRED_HOME}/bin/mapred" "${HADOOP_USER_PARAMS[@]}"
+  exit $?
+fi
+
 daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out"
 daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}.pid"
 

hadoop-yarn-project/hadoop-yarn/bin/yarn

@@ -186,6 +186,11 @@ esac
 
 hadoop_verify_user "${COMMAND}"
 
+if [[ ${HADOOP_SLAVE_MODE} = true ]]; then
+  hadoop_common_slave_mode_execute "${HADOOP_YARN_HOME}/bin/yarn" "${HADOOP_USER_PARAMS[@]}"
+  exit $?
+fi
+
 daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out"
 daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}.pid"
 

hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh

@@ -41,8 +41,23 @@ else
   exit 1
 fi
 
+yarnscript="${HADOOP_YARN_HOME}/bin/yarn"
+
 daemonmode=$1
 shift
 
-hadoop_connect_to_hosts "${HADOOP_YARN_HOME}/bin/yarn" \
-    --config "${HADOOP_CONF_DIR}" --daemon "${daemonmode}" "$@"
+hadoop_error "WARNING: Use of this script to ${daemonmode} YARN daemons is deprecated."
+hadoop_error "WARNING: Attempting to execute replacement \"yarn --slaves --daemon ${daemonmode}\" instead."
+
+#
+# Original input was usually:
+#  yarn-daemons.sh (shell options) (start|stop) nodemanager (daemon options)
+# we're going to turn this into
+#  yarn --slaves --daemon (start|stop) (rest of options)
+#
+argv=(${HADOOP_USER_PARAMS[@]/start})
+argv=(${argv[@]/stop})
+argv=(${argv[@]/status})
+
+${yarnscript} --slaves --daemon "${daemonmode}" "${argv[@]}"
+