HADOOP-11590. Update sbin commands and documentation to use new --slaves option (aw)
This commit is contained in:
Allen Wittenauer
parent
effcc5cbb7
commit
c945c20483
@ -212,6 +212,9 @@ Trunk (Unreleased)
|
||||
HADOOP-11813. releasedocmaker.py should use today's date instead of
|
||||
unreleased (Darrell Taylor via aw)
|
||||
|
||||
HADOOP-11590. Update sbin commands and documentation to use new --slaves
|
||||
option (aw)
|
||||
|
||||
BUG FIXES
|
||||
|
||||
HADOOP-11473. test-patch says "-1 overall" even when all checks are +1
|
||||
|
||||
@ -64,13 +64,14 @@ All of the shell commands will accept a common set of options. For some commands
|
||||
| SHELL\_OPTION | Description |
|
||||
|:---- |:---- |
|
||||
| `--buildpaths` | Enables developer versions of jars. |
|
||||
| `--config confdir` | Overwrites the default Configuration directory. Default is `$HADOOP_PREFIX/conf`. |
|
||||
| `--daemon mode` | If the command supports daemonization (e.g., `hdfs namenode`), execute in the appropriate mode. Supported modes are `start` to start the process in daemon mode, `stop` to stop the process, and `status` to determine the active status of the process. `status` will return an [LSB-compliant](http://refspecs.linuxbase.org/LSB_3.0.0/LSB-generic/LSB-generic/iniscrptact.html) result code. If no option is provided, commands that support daemonization will run in the foreground. |
|
||||
| `--config confdir` | Overwrites the default Configuration directory. Default is `$HADOOP_PREFIX/etc/hadoop`. |
|
||||
| `--daemon mode` | If the command supports daemonization (e.g., `hdfs namenode`), execute in the appropriate mode. Supported modes are `start` to start the process in daemon mode, `stop` to stop the process, and `status` to determine the active status of the process. `status` will return an [LSB-compliant](http://refspecs.linuxbase.org/LSB_3.0.0/LSB-generic/LSB-generic/iniscrptact.html) result code. If no option is provided, commands that support daemonization will run in the foreground. For commands that do not support daemonization, this option is ignored. |
|
||||
| `--debug` | Enables shell level configuration debugging information |
|
||||
| `--help` | Shell script usage information. |
|
||||
| `--hostnames` | A space delimited list of hostnames where to execute a multi-host subcommand. By default, the content of the `slaves` file is used. |
|
||||
| `--hosts` | A file that contains a list of hostnames where to execute a multi-host subcommand. By default, the content of the `slaves` file is used. |
|
||||
| `--hostnames` | When `--slaves` is used, override the slaves file with a space delimited list of hostnames where to execute a multi-host subcommand. If `--slaves` is not used, this option is ignored. |
|
||||
| `--hosts` | When `--slaves` is used, override the slaves file with another file that contains a list of hostnames where to execute a multi-host subcommand. If `--slaves` is not used, this option is ignored. |
|
||||
| `--loglevel loglevel` | Overrides the log level. Valid log levels are FATAL, ERROR, WARN, INFO, DEBUG, and TRACE. Default is INFO. |
|
||||
| `--slaves` | If possible, execute this command on all hosts in the `slaves` file. |
|
||||
|
||||
### Generic Options
|
||||
|
||||
|
||||
@ -65,7 +65,7 @@ fi
|
||||
|
||||
|
||||
#Add other possible options
|
||||
nameStartOpt="$nameStartOpt $@"
|
||||
nameStartOpt="$nameStartOpt $*"
|
||||
|
||||
#---------------------------------------------------------
|
||||
# namenodes
|
||||
@ -76,28 +76,32 @@ if [[ -z "${NAMENODES}" ]]; then
|
||||
NAMENODES=$(hostname)
|
||||
fi
|
||||
|
||||
echo "Starting namenodes on [$NAMENODES]"
|
||||
echo "Starting namenodes on [${NAMENODES}]"
|
||||
|
||||
"${bin}/hadoop-daemons.sh" \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--hostnames "${NAMENODES}" \
|
||||
start namenode ${nameStartOpt}
|
||||
"${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
--slaves \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--hostnames "${NAMENODES}" \
|
||||
--daemon start \
|
||||
namenode ${nameStartOpt}
|
||||
|
||||
#---------------------------------------------------------
|
||||
# datanodes (using default slaves file)
|
||||
|
||||
if [[ -n "${HADOOP_SECURE_DN_USER}" ]] &&
|
||||
[[ -z "${HADOOP_SECURE_COMMAND}" ]]; then
|
||||
echo "ERROR: Attempting to start secure cluster, skipping datanodes. "
|
||||
echo "Run start-secure-dns.sh as root or configure "
|
||||
echo "\${HADOOP_SECURE_COMMAND} to complete startup."
|
||||
[[ -z "${HADOOP_SECURE_COMMAND}" ]]; then
|
||||
hadoop_error "ERROR: Attempting to start secure cluster, skipping datanodes. "
|
||||
hadoop_error "ERROR: Run start-secure-dns.sh as root or configure "
|
||||
hadoop_error "ERROR: \${HADOOP_SECURE_COMMAND} to complete startup."
|
||||
else
|
||||
|
||||
|
||||
echo "Starting datanodes"
|
||||
|
||||
"${bin}/hadoop-daemons.sh" \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
start datanode ${dataStartOpt}
|
||||
|
||||
"${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
--slaves \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--daemon start \
|
||||
datanode ${dataStartOpt}
|
||||
fi
|
||||
|
||||
#---------------------------------------------------------
|
||||
@ -105,17 +109,28 @@ fi
|
||||
|
||||
SECONDARY_NAMENODES=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -secondarynamenodes 2>/dev/null)
|
||||
|
||||
if [[ "${SECONDARY_NAMENODES}" == "0.0.0.0" ]]; then
|
||||
SECONDARY_NAMENODES=$(hostname)
|
||||
fi
|
||||
|
||||
if [[ -n "${SECONDARY_NAMENODES}" ]]; then
|
||||
echo "Starting secondary namenodes [${SECONDARY_NAMENODES}]"
|
||||
|
||||
"${bin}/hadoop-daemons.sh" \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--hostnames "${SECONDARY_NAMENODES}" \
|
||||
start secondarynamenode
|
||||
|
||||
if [[ "${NAMENODES}" =~ , ]]; then
|
||||
|
||||
hadoop_error "ERROR: Highly available NameNode is configured."
|
||||
hadoop_error "ERROR: Skipping SecondaryNameNode."
|
||||
|
||||
else
|
||||
|
||||
if [[ "${SECONDARY_NAMENODES}" == "0.0.0.0" ]]; then
|
||||
SECONDARY_NAMENODES=$(hostname)
|
||||
fi
|
||||
|
||||
echo "Starting secondary namenodes [${SECONDARY_NAMENODES}]"
|
||||
|
||||
"${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
--slaves \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--hostnames "${SECONDARY_NAMENODES}" \
|
||||
--daemon start \
|
||||
secondarynamenode
|
||||
fi
|
||||
fi
|
||||
|
||||
#---------------------------------------------------------
|
||||
@ -127,10 +142,13 @@ case "${SHARED_EDITS_DIR}" in
|
||||
qjournal://*)
|
||||
JOURNAL_NODES=$(echo "${SHARED_EDITS_DIR}" | sed 's,qjournal://\([^/]*\)/.*,\1,g; s/;/ /g; s/:[0-9]*//g')
|
||||
echo "Starting journal nodes [${JOURNAL_NODES}]"
|
||||
"${bin}/hadoop-daemons.sh" \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--hostnames "${JOURNAL_NODES}" \
|
||||
start journalnode
|
||||
|
||||
"${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
--slaves \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--hostnames "${JOURNAL_NODES}" \
|
||||
--daemon start \
|
||||
journalnode
|
||||
;;
|
||||
esac
|
||||
|
||||
@ -139,10 +157,13 @@ esac
|
||||
AUTOHA_ENABLED=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey dfs.ha.automatic-failover.enabled | tr '[:upper:]' '[:lower:]')
|
||||
if [[ "${AUTOHA_ENABLED}" = "true" ]]; then
|
||||
echo "Starting ZK Failover Controllers on NN hosts [${NAMENODES}]"
|
||||
"${bin}/hadoop-daemons.sh" \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--hostnames "${NAMENODES}" \
|
||||
start zkfc
|
||||
|
||||
"${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
--slaves \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--hostnames "${NAMENODES}" \
|
||||
--daemon start \
|
||||
zkfc
|
||||
fi
|
||||
|
||||
# eof
|
||||
|
||||
@ -43,7 +43,11 @@ else
|
||||
fi
|
||||
|
||||
if [[ "${EUID}" -eq 0 ]] && [[ -n "${HADOOP_SECURE_DN_USER}" ]]; then
|
||||
exec "${bin}/hadoop-daemons.sh" --config "${HADOOP_CONF_DIR}" start datanode "${dataStartOpt}"
|
||||
exec "${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--slaves \
|
||||
--daemon start \
|
||||
datanode
|
||||
else
|
||||
echo hadoop_usage_and_exit 1
|
||||
fi
|
||||
|
||||
@ -15,9 +15,13 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
# Stop hadoop dfs daemons.
|
||||
# Run this on master node.
|
||||
|
||||
function hadoop_usage
|
||||
{
|
||||
echo "Usage: start-balancer.sh [--config confdir] [-policy <policy>] [-threshold <threshold>]"
|
||||
echo "Usage: stop-dfs.sh [-upgrade|-rollback] [-clusterId]"
|
||||
}
|
||||
|
||||
this="${BASH_SOURCE-$0}"
|
||||
@ -43,28 +47,38 @@ fi
|
||||
#---------------------------------------------------------
|
||||
# namenodes
|
||||
|
||||
NAMENODES=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -namenodes)
|
||||
NAMENODES=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -namenodes 2>/dev/null)
|
||||
|
||||
echo "Stopping namenodes on [$NAMENODES]"
|
||||
if [[ -z "${NAMENODES}" ]]; then
|
||||
NAMENODES=$(hostname)
|
||||
fi
|
||||
|
||||
"${bin}/hadoop-daemons.sh" \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--hostnames "${NAMENODES}" \
|
||||
stop namenode
|
||||
echo "Stopping namenodes on [${NAMENODES}]"
|
||||
|
||||
"${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
--slaves \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--hostnames "${NAMENODES}" \
|
||||
--daemon stop \
|
||||
namenode
|
||||
|
||||
#---------------------------------------------------------
|
||||
# datanodes (using default slaves file)
|
||||
|
||||
if [[ -n "${HADOOP_SECURE_DN_USER}" ]] &&
|
||||
[[ -z "${HADOOP_SECURE_COMMAND}" ]]; then
|
||||
echo \
|
||||
"ERROR: Attempting to stop secure cluster, skipping datanodes. " \
|
||||
"Run stop-secure-dns.sh as root to complete shutdown."
|
||||
echo "ERROR: Attempting to stop secure cluster, skipping datanodes. "
|
||||
echo "Run stop-secure-dns.sh as root or configure "
|
||||
echo "\${HADOOP_SECURE_COMMAND} to complete stop."
|
||||
else
|
||||
|
||||
|
||||
echo "Stopping datanodes"
|
||||
|
||||
"${bin}/hadoop-daemons.sh" --config "${HADOOP_CONF_DIR}" stop datanode
|
||||
|
||||
"${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
--slaves \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--daemon stop \
|
||||
datanode
|
||||
fi
|
||||
|
||||
#---------------------------------------------------------
|
||||
@ -78,11 +92,13 @@ fi
|
||||
|
||||
if [[ -n "${SECONDARY_NAMENODES}" ]]; then
|
||||
echo "Stopping secondary namenodes [${SECONDARY_NAMENODES}]"
|
||||
|
||||
"${bin}/hadoop-daemons.sh" \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--hostnames "${SECONDARY_NAMENODES}" \
|
||||
stop secondarynamenode
|
||||
|
||||
"${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
--slaves \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--hostnames "${SECONDARY_NAMENODES}" \
|
||||
--daemon stop \
|
||||
secondarynamenode
|
||||
fi
|
||||
|
||||
#---------------------------------------------------------
|
||||
@ -94,10 +110,13 @@ case "${SHARED_EDITS_DIR}" in
|
||||
qjournal://*)
|
||||
JOURNAL_NODES=$(echo "${SHARED_EDITS_DIR}" | sed 's,qjournal://\([^/]*\)/.*,\1,g; s/;/ /g; s/:[0-9]*//g')
|
||||
echo "Stopping journal nodes [${JOURNAL_NODES}]"
|
||||
"${bin}/hadoop-daemons.sh" \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--hostnames "${JOURNAL_NODES}" \
|
||||
stop journalnode
|
||||
|
||||
"${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
--slaves \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--hostnames "${JOURNAL_NODES}" \
|
||||
--daemon stop \
|
||||
journalnode
|
||||
;;
|
||||
esac
|
||||
|
||||
@ -106,9 +125,13 @@ esac
|
||||
AUTOHA_ENABLED=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey dfs.ha.automatic-failover.enabled | tr '[:upper:]' '[:lower:]')
|
||||
if [[ "${AUTOHA_ENABLED}" = "true" ]]; then
|
||||
echo "Stopping ZK Failover Controllers on NN hosts [${NAMENODES}]"
|
||||
"${bin}/hadoop-daemons.sh" \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--hostnames "${NAMENODES}" \
|
||||
stop zkfc
|
||||
|
||||
"${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
--slaves \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--hostnames "${NAMENODES}" \
|
||||
--daemon stop \
|
||||
zkfc
|
||||
fi
|
||||
|
||||
# eof
|
||||
|
||||
@ -15,11 +15,11 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Run as root to start secure datanodes in a security-enabled cluster.
|
||||
# Run as root to stop secure datanodes in a security-enabled cluster.
|
||||
|
||||
|
||||
function hadoop_usage {
|
||||
echo "Usage (run as root in order to stop secure datanodes): stop-secure-dns.sh"
|
||||
echo "Usage: stop-secure-dns.sh"
|
||||
}
|
||||
|
||||
this="${BASH_SOURCE-$0}"
|
||||
@ -43,7 +43,11 @@ else
|
||||
fi
|
||||
|
||||
if [[ "${EUID}" -eq 0 ]] && [[ -n "${HADOOP_SECURE_DN_USER}" ]]; then
|
||||
"${bin}/hadoop-daemons.sh" --config "${HADOOP_CONF_DIR}" stop datanode
|
||||
exec "${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--slaves \
|
||||
--daemon stop \
|
||||
datanode
|
||||
else
|
||||
hadoop_exit_with_usage 1
|
||||
echo hadoop_usage_and_exit 1
|
||||
fi
|
||||
|
||||
@ -355,7 +355,7 @@ The order in which you set these configurations is unimportant, but the values y
|
||||
|
||||
### Deployment details
|
||||
|
||||
After all of the necessary configuration options have been set, you must start the JournalNode daemons on the set of machines where they will run. This can be done by running the command "*hadoop-daemon.sh start journalnode*" and waiting for the daemon to start on each of the relevant machines.
|
||||
After all of the necessary configuration options have been set, you must start the JournalNode daemons on the set of machines where they will run. This can be done by running the command "*hdfs --daemon start journalnode*" and waiting for the daemon to start on each of the relevant machines.
|
||||
|
||||
Once the JournalNodes have been started, one must initially synchronize the two HA NameNodes' on-disk metadata.
|
||||
|
||||
|
||||
@ -42,10 +42,46 @@ else
|
||||
fi
|
||||
|
||||
# start resourceManager
|
||||
echo "Starting resourcemanager"
|
||||
"${HADOOP_YARN_HOME}/bin/yarn" --config "${HADOOP_CONF_DIR}" --daemon start resourcemanager
|
||||
# start nodeManager
|
||||
echo "Starting nodemanagers"
|
||||
"${bin}/yarn-daemons.sh" --config "${HADOOP_CONF_DIR}" start nodemanager
|
||||
HARM=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey yarn.resourcemanager.ha.enabled 2>&-)
|
||||
if [[ ${HARM} = "false" ]]; then
|
||||
echo "Starting resourcemanager"
|
||||
"${HADOOP_YARN_HOME}/bin/yarn" \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--daemon start \
|
||||
resourcemanager
|
||||
else
|
||||
logicals=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey yarn.resourcemanager.ha.rm-ids 2>&-)
|
||||
logicals=${logicals//,/ }
|
||||
for id in ${logicals}
|
||||
do
|
||||
rmhost=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey "yarn.resourcemanager.hostname.${id}" 2>&-)
|
||||
RMHOSTS="${RMHOSTS} ${rmhost}"
|
||||
done
|
||||
echo "Starting resourcemanagers on [${RMHOSTS}]"
|
||||
"${HADOOP_YARN_HOME}/bin/yarn" \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--daemon start \
|
||||
--slaves \
|
||||
--hostnames "${RMHOSTS}" \
|
||||
resourcemanager
|
||||
fi
|
||||
|
||||
# start nodemanager
|
||||
echo "Starting nodemanagers"
|
||||
"${HADOOP_YARN_HOME}/bin/yarn" \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--slaves \
|
||||
--daemon start \
|
||||
nodemanager
|
||||
|
||||
# start proxyserver
|
||||
#"${HADOOP_YARN_HOME}/bin/yarn" --config "${HADOOP_CONF_DIR}" --daemon start proxyserver
|
||||
PROXYSERVER=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey yarn.web-proxy.address 2>&- | cut -f1 -d:)
|
||||
if [[ -n ${PROXYSERVER} ]]; then
|
||||
"${HADOOP_YARN_HOME}/bin/yarn" \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--slaves \
|
||||
--hostnames "${PROXYSERVER}" \
|
||||
--daemon start \
|
||||
proxyserver
|
||||
fi
|
||||
|
||||
|
||||
@ -16,8 +16,6 @@
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
# Stop all yarn daemons. Run this on master node.
|
||||
|
||||
function hadoop_usage
|
||||
{
|
||||
echo "Usage: stop-yarn.sh [--config confdir]"
|
||||
@ -44,10 +42,47 @@ else
|
||||
fi
|
||||
|
||||
# stop resourceManager
|
||||
echo "Stopping resourcemanager"
|
||||
"${HADOOP_YARN_HOME}/bin/yarn" --config "${HADOOP_CONF_DIR}" --daemon stop resourcemanager
|
||||
# stop nodeManager
|
||||
HARM=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey yarn.resourcemanager.ha.enabled 2>&-)
|
||||
if [[ ${HARM} = "false" ]]; then
|
||||
echo "Stopping resourcemanager"
|
||||
"${HADOOP_YARN_HOME}/bin/yarn" \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--daemon stop \
|
||||
resourcemanager
|
||||
else
|
||||
logicals=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey yarn.resourcemanager.ha.rm-ids 2>&-)
|
||||
logicals=${logicals//,/ }
|
||||
for id in ${logicals}
|
||||
do
|
||||
rmhost=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey "yarn.resourcemanager.hostname.${id}" 2>&-)
|
||||
RMHOSTS="${RMHOSTS} ${rmhost}"
|
||||
done
|
||||
echo "Stopping resourcemanagers on [${RMHOSTS}]"
|
||||
"${HADOOP_YARN_HOME}/bin/yarn" \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--daemon stop \
|
||||
--slaves \
|
||||
--hostnames "${RMHOSTS}" \
|
||||
resourcemanager
|
||||
fi
|
||||
|
||||
# stop nodemanager
|
||||
echo "Stopping nodemanagers"
|
||||
"${bin}/yarn-daemons.sh" --config "${HADOOP_CONF_DIR}" stop nodemanager
|
||||
"${HADOOP_YARN_HOME}/bin/yarn" \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--slaves \
|
||||
--daemon stop \
|
||||
nodemanager
|
||||
|
||||
# stop proxyserver
|
||||
#"${HADOOP_YARN_HOME}/bin/yarn" --config "${HADOOP_CONF_DIR}" --daemon stop proxyserver
|
||||
PROXYSERVER=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey yarn.web-proxy.address 2>&- | cut -f1 -d:)
|
||||
if [[ -n ${PROXYSERVER} ]]; then
|
||||
echo "Stopping proxy server [${PROXYSERVER}]"
|
||||
"${HADOOP_YARN_HOME}/bin/yarn" \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--slaves \
|
||||
--hostnames "${PROXYSERVER}" \
|
||||
--daemon stop \
|
||||
proxyserver
|
||||
fi
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user