58ed4fa544
This commit includes the following changes: HADOOP-13356. Add a function to handle command_subcommand_OPTS HADOOP-13355. Handle HADOOP_CLIENT_OPTS in a function HADOOP-13554. Add an equivalent of hadoop_subcmd_opts for secure opts HADOOP-13562. Change hadoop_subcommand_opts to use only uppercase HADOOP-13358. Modify HDFS to use hadoop_subcommand_opts HADOOP-13357. Modify common to use hadoop_subcommand_opts HADOOP-13359. Modify YARN to use hadoop_subcommand_opts HADOOP-13361. Modify hadoop_verify_user to be consistent with hadoop_subcommand_opts (ie more granularity) HADOOP-13564. modify mapred to use hadoop_subcommand_opts HADOOP-13563. hadoop_subcommand_opts should print name not actual content during debug HADOOP-13360. Documentation for HADOOP_subcommand_OPTS This closes apache/hadoop#126
313 lines
11 KiB
Bash
Executable File
313 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
# contributor license agreements. See the NOTICE file distributed with
|
|
# this work for additional information regarding copyright ownership.
|
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
# (the "License"); you may not use this file except in compliance with
|
|
# the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
MYNAME="${BASH_SOURCE-$0}"
|
|
HADOOP_SHELL_EXECNAME="${MYNAME##*/}"
|
|
|
|
## @description build up the yarn command's usage text.
|
|
## @audience public
|
|
## @stability stable
|
|
## @replaceable no
|
|
function hadoop_usage
|
|
{
|
|
hadoop_add_option "--buildpaths" "attempt to add class files from build tree"
|
|
hadoop_add_option "--daemon (start|status|stop)" "operate on a daemon"
|
|
hadoop_add_option "--hostnames list[,of,host,names]" "hosts to use in worker mode"
|
|
hadoop_add_option "--loglevel level" "set the log4j level for this command"
|
|
hadoop_add_option "--hosts filename" "list of hosts to use in worker mode"
|
|
hadoop_add_option "--workers" "turn on worker mode"
|
|
|
|
hadoop_add_subcommand "application" "prints application(s) report/kill application"
|
|
hadoop_add_subcommand "applicationattempt" "prints applicationattempt(s) report"
|
|
hadoop_add_subcommand "classpath" "prints the class path needed to get the hadoop jar and the required libraries"
|
|
hadoop_add_subcommand "cluster" "prints cluster information"
|
|
hadoop_add_subcommand "container" "prints container(s) report"
|
|
hadoop_add_subcommand "daemonlog" "get/set the log level for each daemon"
|
|
hadoop_add_subcommand "envvars" "display computed Hadoop environment variables"
|
|
hadoop_add_subcommand "jar <jar>" "run a jar file"
|
|
hadoop_add_subcommand "logs" "dump container logs"
|
|
hadoop_add_subcommand "node" "prints node report(s)"
|
|
hadoop_add_subcommand "nodemanager" "run a nodemanager on each worker"
|
|
hadoop_add_subcommand "proxyserver" "run the web app proxy server"
|
|
hadoop_add_subcommand "queue" "prints queue information"
|
|
hadoop_add_subcommand "resourcemanager" "run the ResourceManager"
|
|
hadoop_add_subcommand "rmadmin" "admin tools"
|
|
hadoop_add_subcommand "scmadmin" "SharedCacheManager admin tools"
|
|
hadoop_add_subcommand "sharedcachemanager" "run the SharedCacheManager daemon"
|
|
hadoop_add_subcommand "timelinereader" "run the timeline reader server"
|
|
hadoop_add_subcommand "timelineserver" "run the timeline server"
|
|
hadoop_add_subcommand "top" "view cluster information"
|
|
hadoop_add_subcommand "version" "print the version"
|
|
hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" true
|
|
}
|
|
|
|
## @description Default command handler for yarn command
|
|
## @audience public
|
|
## @stability stable
|
|
## @replaceable no
|
|
## @param CLI arguments
|
|
function yarncmd_case
|
|
{
|
|
subcmd=$1
|
|
shift
|
|
|
|
case ${subcmd} in
|
|
application|applicationattempt|container)
|
|
HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.ApplicationCLI
|
|
set -- "${subcmd}" "$@"
|
|
HADOOP_SUBCMD_ARGS=("$@")
|
|
;;
|
|
classpath)
|
|
hadoop_do_classpath_subcommand HADOOP_CLASSNAME "$@"
|
|
;;
|
|
cluster)
|
|
HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.ClusterCLI
|
|
;;
|
|
daemonlog)
|
|
HADOOP_CLASSNAME=org.apache.hadoop.log.LogLevel
|
|
;;
|
|
envvars)
|
|
echo "JAVA_HOME='${JAVA_HOME}'"
|
|
echo "HADOOP_YARN_HOME='${HADOOP_YARN_HOME}'"
|
|
echo "YARN_DIR='${YARN_DIR}'"
|
|
echo "YARN_LIB_JARS_DIR='${YARN_LIB_JARS_DIR}'"
|
|
echo "HADOOP_CONF_DIR='${HADOOP_CONF_DIR}'"
|
|
echo "HADOOP_TOOLS_HOME='${HADOOP_TOOLS_HOME}'"
|
|
echo "HADOOP_TOOLS_DIR='${HADOOP_TOOLS_DIR}'"
|
|
echo "HADOOP_TOOLS_LIB_JARS_DIR='${HADOOP_TOOLS_LIB_JARS_DIR}'"
|
|
exit 0
|
|
;;
|
|
jar)
|
|
HADOOP_CLASSNAME=org.apache.hadoop.util.RunJar
|
|
;;
|
|
historyserver)
|
|
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
|
|
echo "DEPRECATED: Use of this command to start the timeline server is deprecated." 1>&2
|
|
echo "Instead use the timelineserver command for it." 1>&2
|
|
echo "Starting the History Server anyway..." 1>&2
|
|
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'
|
|
;;
|
|
logs)
|
|
HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.LogsCLI
|
|
;;
|
|
node)
|
|
HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.NodeCLI
|
|
;;
|
|
nodemanager)
|
|
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
|
|
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.nodemanager.NodeManager'
|
|
# Backwards compatibility
|
|
if [[ -n "${YARN_NODEMANAGER_HEAPSIZE}" ]]; then
|
|
HADOOP_HEAPSIZE_MAX="${YARN_NODEMANAGER_HEAPSIZE}"
|
|
fi
|
|
;;
|
|
proxyserver)
|
|
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
|
|
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer'
|
|
# Backwards compatibility
|
|
if [[ -n "${YARN_PROXYSERVER_HEAPSIZE}" ]]; then
|
|
# shellcheck disable=SC2034
|
|
HADOOP_HEAPSIZE_MAX="${YARN_PROXYSERVER_HEAPSIZE}"
|
|
fi
|
|
;;
|
|
queue)
|
|
HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.QueueCLI
|
|
;;
|
|
resourcemanager)
|
|
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
|
|
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager'
|
|
# Backwards compatibility
|
|
if [[ -n "${YARN_RESOURCEMANAGER_HEAPSIZE}" ]]; then
|
|
# shellcheck disable=SC2034
|
|
HADOOP_HEAPSIZE_MAX="${YARN_RESOURCEMANAGER_HEAPSIZE}"
|
|
fi
|
|
;;
|
|
rmadmin)
|
|
HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.cli.RMAdminCLI'
|
|
;;
|
|
scmadmin)
|
|
HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.SCMAdmin'
|
|
;;
|
|
sharedcachemanager)
|
|
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
|
|
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager'
|
|
;;
|
|
timelinereader)
|
|
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
|
|
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer'
|
|
;;
|
|
timelineserver)
|
|
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
|
|
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'
|
|
# Backwards compatibility
|
|
if [[ -n "${YARN_TIMELINESERVER_HEAPSIZE}" ]]; then
|
|
# shellcheck disable=SC2034
|
|
HADOOP_HEAPSIZE_MAX="${YARN_TIMELINESERVER_HEAPSIZE}"
|
|
fi
|
|
;;
|
|
version)
|
|
HADOOP_CLASSNAME=org.apache.hadoop.util.VersionInfo
|
|
;;
|
|
top)
|
|
doNotSetCols=0
|
|
doNotSetRows=0
|
|
for i in "$@"; do
|
|
if [[ $i == "-cols" ]]; then
|
|
doNotSetCols=1
|
|
fi
|
|
if [[ $i == "-rows" ]]; then
|
|
doNotSetRows=1
|
|
fi
|
|
done
|
|
if [ $doNotSetCols == 0 ] && [ -n "${TERM}" ]; then
|
|
cols=$(tput cols)
|
|
if [ -n "$cols" ]; then
|
|
args=( $@ )
|
|
args=("${args[@]}" "-cols" "$cols")
|
|
set -- "${args[@]}"
|
|
fi
|
|
fi
|
|
if [ $doNotSetRows == 0 ] && [ -n "${TERM}" ]; then
|
|
rows=$(tput lines)
|
|
if [ -n "$rows" ]; then
|
|
args=( $@ )
|
|
args=("${args[@]}" "-rows" "$rows")
|
|
set -- "${args[@]}"
|
|
fi
|
|
fi
|
|
HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.TopCLI
|
|
HADOOP_SUBCMD_ARGS=("$@")
|
|
;;
|
|
*)
|
|
HADOOP_CLASSNAME="${subcmd}"
|
|
if ! hadoop_validate_classname "${HADOOP_CLASSNAME}"; then
|
|
hadoop_exit_with_usage 1
|
|
fi
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# let's locate libexec...
|
|
if [[ -n "${HADOOP_HOME}" ]]; then
|
|
HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
|
|
else
|
|
bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P)
|
|
HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
|
|
fi
|
|
|
|
HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}"
|
|
# shellcheck disable=SC2034
|
|
HADOOP_NEW_CONFIG=true
|
|
if [[ -f "${HADOOP_LIBEXEC_DIR}/yarn-config.sh" ]]; then
|
|
. "${HADOOP_LIBEXEC_DIR}/yarn-config.sh"
|
|
else
|
|
echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/yarn-config.sh." 2>&1
|
|
exit 1
|
|
fi
|
|
|
|
# if no args specified, show usage
|
|
if [[ $# = 0 ]]; then
|
|
hadoop_exit_with_usage 1
|
|
fi
|
|
|
|
# get arguments
|
|
HADOOP_SUBCMD=$1
|
|
shift
|
|
|
|
hadoop_verify_user "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}"
|
|
|
|
HADOOP_SUBCMD_ARGS=("$@")
|
|
|
|
if declare -f yarn_subcommand_"${HADOOP_SUBCMD}" >/dev/null 2>&1; then
|
|
hadoop_debug "Calling dynamically: yarn_subcommand_${HADOOP_SUBCMD} ${HADOOP_SUBCMD_ARGS[*]}"
|
|
"yarn_subcommand_${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}"
|
|
else
|
|
yarncmd_case "${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}"
|
|
fi
|
|
|
|
# It's unclear if YARN_CLIENT_OPTS is actually a useful
|
|
# thing to have separate from HADOOP_CLIENT_OPTS. Someone
|
|
# might use it, so let's not deprecate it and just override
|
|
# HADOOP_CLIENT_OPTS instead before we (potentially) add it
|
|
# to the command line
|
|
if [[ -n "${YARN_CLIENT_OPTS}" ]]; then
|
|
# shellcheck disable=SC2034
|
|
HADOOP_CLIENT_OPTS=${YARN_CLIENT_OPTS}
|
|
fi
|
|
|
|
hadoop_add_client_opts
|
|
|
|
if [[ ${HADOOP_WORKER_MODE} = true ]]; then
|
|
hadoop_common_worker_mode_execute "${HADOOP_YARN_HOME}/bin/yarn" "${HADOOP_USER_PARAMS[@]}"
|
|
exit $?
|
|
fi
|
|
|
|
hadoop_subcommand_opts "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}"
|
|
|
|
if [[ "${HADOOP_SUBCMD_SECURESERVICE}" = true ]]; then
|
|
HADOOP_SECURE_USER="${HADOOP_SUBCMD_SECUREUSER}"
|
|
|
|
hadoop_subcommand_secure_opts "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}"
|
|
|
|
hadoop_verify_secure_prereq
|
|
hadoop_setup_secure_service
|
|
priv_outfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out"
|
|
priv_errfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.err"
|
|
priv_pidfile="${HADOOP_PID_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid"
|
|
daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out"
|
|
daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid"
|
|
else
|
|
daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out"
|
|
daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid"
|
|
fi
|
|
|
|
if [[ "${HADOOP_DAEMON_MODE}" != "default" ]]; then
|
|
# shellcheck disable=SC2034
|
|
HADOOP_ROOT_LOGGER="${HADOOP_DAEMON_ROOT_LOGGER}"
|
|
# shellcheck disable=SC2034
|
|
HADOOP_LOGFILE="hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.log"
|
|
fi
|
|
|
|
hadoop_finalize
|
|
|
|
if [[ "${HADOOP_SUBCMD_SUPPORTDAEMONIZATION}" = true ]]; then
|
|
if [[ "${HADOOP_SUBCMD_SECURESERVICE}" = true ]]; then
|
|
hadoop_secure_daemon_handler \
|
|
"${HADOOP_DAEMON_MODE}" \
|
|
"${HADOOP_SUBCMD}" \
|
|
"${HADOOP_CLASSNAME}" \
|
|
"${daemon_pidfile}" \
|
|
"${daemon_outfile}" \
|
|
"${priv_pidfile}" \
|
|
"${priv_outfile}" \
|
|
"${priv_errfile}" \
|
|
"${HADOOP_SUBCMD_ARGS[@]}"
|
|
else
|
|
hadoop_daemon_handler \
|
|
"${HADOOP_DAEMON_MODE}" \
|
|
"${HADOOP_SUBCMD}" \
|
|
"${HADOOP_CLASSNAME}" \
|
|
"${daemon_pidfile}" \
|
|
"${daemon_outfile}" \
|
|
"${HADOOP_SUBCMD_ARGS[@]}"
|
|
fi
|
|
exit $?
|
|
else
|
|
# shellcheck disable=SC2086
|
|
hadoop_java_exec "${HADOOP_SUBCMD}" "${HADOOP_CLASSNAME}" "${HADOOP_SUBCMD_ARGS[@]}"
|
|
fi
|