#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. MYNAME="${BASH_SOURCE-$0}" HADOOP_SHELL_EXECNAME="${MYNAME##*/}" ## @description build up the yarn command's usage text. ## @audience public ## @stability stable ## @replaceable no function hadoop_usage { hadoop_add_option "--buildpaths" "attempt to add class files from build tree" hadoop_add_option "--daemon (start|status|stop)" "operate on a daemon" hadoop_add_option "--hostnames list[,of,host,names]" "hosts to use in worker mode" hadoop_add_option "--loglevel level" "set the log4j level for this command" hadoop_add_option "--hosts filename" "list of hosts to use in worker mode" hadoop_add_option "--workers" "turn on worker mode" hadoop_add_subcommand "application" client "prints application(s) report/kill application" hadoop_add_subcommand "applicationattempt" client "prints applicationattempt(s) report" hadoop_add_subcommand "classpath" client "prints the class path needed to get the hadoop jar and the required libraries" hadoop_add_subcommand "cluster" client "prints cluster information" hadoop_add_subcommand "container" client "prints container(s) report" hadoop_add_subcommand "daemonlog" admin "get/set the log level for each daemon" hadoop_add_subcommand "envvars" client "display computed Hadoop environment variables" hadoop_add_subcommand "jar " client "run a jar file" hadoop_add_subcommand "logs" client "dump container logs" hadoop_add_subcommand "node" admin "prints node report(s)" hadoop_add_subcommand "nodemanager" daemon "run a nodemanager on each worker" hadoop_add_subcommand "proxyserver" daemon "run the web app proxy server" hadoop_add_subcommand "queue" client "prints queue information" hadoop_add_subcommand "resourcemanager" daemon "run the ResourceManager" hadoop_add_subcommand "rmadmin" admin "admin tools" hadoop_add_subcommand "router" daemon "run the Router daemon" hadoop_add_subcommand "schedulerconf" client "Updates scheduler configuration" hadoop_add_subcommand "scmadmin" admin "SharedCacheManager admin tools" hadoop_add_subcommand "servicesapi" "run yarn-service rest server" hadoop_add_subcommand "sharedcachemanager" daemon "run the SharedCacheManager daemon" hadoop_add_subcommand "service" "run a service" hadoop_add_subcommand "timelinereader" client "run the timeline reader server" hadoop_add_subcommand "timelineserver" daemon "run the timeline server" hadoop_add_subcommand "top" client "view cluster information" hadoop_add_subcommand "version" client "print the version" hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" true } ## @description Default command handler for yarn command ## @audience public ## @stability stable ## @replaceable no ## @param CLI arguments function yarncmd_case { subcmd=$1 shift case ${subcmd} in application|applicationattempt|container) HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.ApplicationCLI set -- "${subcmd}" "$@" HADOOP_SUBCMD_ARGS=("$@") ;; classpath) hadoop_do_classpath_subcommand HADOOP_CLASSNAME "$@" ;; cluster) HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.ClusterCLI ;; daemonlog) HADOOP_CLASSNAME=org.apache.hadoop.log.LogLevel ;; envvars) echo "JAVA_HOME='${JAVA_HOME}'" echo "HADOOP_YARN_HOME='${HADOOP_YARN_HOME}'" echo "YARN_DIR='${YARN_DIR}'" echo "YARN_LIB_JARS_DIR='${YARN_LIB_JARS_DIR}'" echo "HADOOP_CONF_DIR='${HADOOP_CONF_DIR}'" echo "HADOOP_TOOLS_HOME='${HADOOP_TOOLS_HOME}'" echo "HADOOP_TOOLS_DIR='${HADOOP_TOOLS_DIR}'" echo "HADOOP_TOOLS_LIB_JARS_DIR='${HADOOP_TOOLS_LIB_JARS_DIR}'" exit 0 ;; jar) HADOOP_CLASSNAME=org.apache.hadoop.util.RunJar ;; historyserver) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" echo "DEPRECATED: Use of this command to start the timeline server is deprecated." 1>&2 echo "Instead use the timelineserver command for it." 1>&2 echo "Starting the History Server anyway..." 1>&2 HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer' ;; logs) HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.LogsCLI ;; node) HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.NodeCLI ;; nodemanager) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.nodemanager.NodeManager' # Backwards compatibility if [[ -n "${YARN_NODEMANAGER_HEAPSIZE}" ]]; then HADOOP_HEAPSIZE_MAX="${YARN_NODEMANAGER_HEAPSIZE}" fi ;; proxyserver) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer' # Backwards compatibility if [[ -n "${YARN_PROXYSERVER_HEAPSIZE}" ]]; then HADOOP_HEAPSIZE_MAX="${YARN_PROXYSERVER_HEAPSIZE}" fi ;; queue) HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.QueueCLI ;; resourcemanager) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager' # Backwards compatibility if [[ -n "${YARN_RESOURCEMANAGER_HEAPSIZE}" ]]; then HADOOP_HEAPSIZE_MAX="${YARN_RESOURCEMANAGER_HEAPSIZE}" fi ;; rmadmin) HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.cli.RMAdminCLI' ;; router) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.router.Router' ;; schedulerconf) HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.cli.SchedConfCLI' ;; scmadmin) HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.SCMAdmin' ;; servicesapi) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*' hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services-api"'/*' HADOOP_CLASSNAME='org.apache.hadoop.yarn.services.webapp.ApplicationApiWebApp' local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\ ${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\ ${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\ ${HADOOP_HDFS_HOME}/${HDFS_DIR},\ ${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}" hadoop_translate_cygwin_path sld hadoop_add_param HADOOP_OPTS slider.libdir "-Dslider.libdir=${sld}" ;; sharedcachemanager) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager' ;; service) hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*' HADOOP_CLASSNAME='org.apache.hadoop.yarn.service.client.ServiceCLI' local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\ ${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\ ${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\ ${HADOOP_HDFS_HOME}/${HDFS_DIR},\ ${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}" hadoop_translate_cygwin_path sld hadoop_add_param HADOOP_OPTS slider.libdir "-Dslider.libdir=${sld}" ;; timelinereader) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer' ;; timelineserver) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer' # Backwards compatibility if [[ -n "${YARN_TIMELINESERVER_HEAPSIZE}" ]]; then HADOOP_HEAPSIZE_MAX="${YARN_TIMELINESERVER_HEAPSIZE}" fi ;; version) HADOOP_CLASSNAME=org.apache.hadoop.util.VersionInfo ;; top) doNotSetCols=0 doNotSetRows=0 for i in "$@"; do if [[ $i == "-cols" ]]; then doNotSetCols=1 fi if [[ $i == "-rows" ]]; then doNotSetRows=1 fi done if [ $doNotSetCols == 0 ] && [ -n "${TERM}" ]; then cols=$(tput cols) if [ -n "$cols" ]; then args=( $@ ) args=("${args[@]}" "-cols" "$cols") set -- "${args[@]}" fi fi if [ $doNotSetRows == 0 ] && [ -n "${TERM}" ]; then rows=$(tput lines) if [ -n "$rows" ]; then args=( $@ ) args=("${args[@]}" "-rows" "$rows") set -- "${args[@]}" fi fi HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.TopCLI HADOOP_SUBCMD_ARGS=("$@") ;; *) HADOOP_CLASSNAME="${subcmd}" if ! hadoop_validate_classname "${HADOOP_CLASSNAME}"; then hadoop_exit_with_usage 1 fi ;; esac } # let's locate libexec... if [[ -n "${HADOOP_HOME}" ]]; then HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec" else bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P) HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec" fi HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}" HADOOP_NEW_CONFIG=true if [[ -f "${HADOOP_LIBEXEC_DIR}/yarn-config.sh" ]]; then # shellcheck source=./hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh . "${HADOOP_LIBEXEC_DIR}/yarn-config.sh" else echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/yarn-config.sh." 2>&1 exit 1 fi # now that we have support code, let's abs MYNAME so we can use it later MYNAME=$(hadoop_abs "${MYNAME}") # if no args specified, show usage if [[ $# = 0 ]]; then hadoop_exit_with_usage 1 fi # get arguments HADOOP_SUBCMD=$1 shift if hadoop_need_reexec yarn "${HADOOP_SUBCMD}"; then hadoop_uservar_su yarn "${HADOOP_SUBCMD}" \ "${MYNAME}" \ "--reexec" \ "${HADOOP_USER_PARAMS[@]}" exit $? fi hadoop_verify_user_perm "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}" HADOOP_SUBCMD_ARGS=("$@") if declare -f yarn_subcommand_"${HADOOP_SUBCMD}" >/dev/null 2>&1; then hadoop_debug "Calling dynamically: yarn_subcommand_${HADOOP_SUBCMD} ${HADOOP_SUBCMD_ARGS[*]}" "yarn_subcommand_${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}" else yarncmd_case "${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}" fi # It's unclear if YARN_CLIENT_OPTS is actually a useful # thing to have separate from HADOOP_CLIENT_OPTS. Someone # might use it, so let's not deprecate it and just override # HADOOP_CLIENT_OPTS instead before we (potentially) add it # to the command line if [[ -n "${YARN_CLIENT_OPTS}" ]]; then HADOOP_CLIENT_OPTS=${YARN_CLIENT_OPTS} fi hadoop_add_client_opts if [[ ${HADOOP_WORKER_MODE} = true ]]; then hadoop_common_worker_mode_execute "${HADOOP_YARN_HOME}/bin/yarn" "${HADOOP_USER_PARAMS[@]}" exit $? fi hadoop_subcommand_opts "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}" # everything is in globals at this point, so call the generic handler hadoop_generic_java_subcmd_handler