hadoop/hadoop-yarn-project/hadoop-yarn/bin/yarn

#!/usr/bin/env bash

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# The name of the script being executed.
HADOOP_SHELL_EXECNAME="yarn"
MYNAME="${BASH_SOURCE-$0}"

## @description  build up the yarn command's usage text.
## @audience     public
## @stability    stable
## @replaceable  no
function hadoop_usage
{
  hadoop_add_option "--buildpaths" "attempt to add class files from build tree"
  hadoop_add_option "--daemon (start|status|stop)" "operate on a daemon"
  hadoop_add_option "--hostnames list[,of,host,names]" "hosts to use in worker mode"
  hadoop_add_option "--loglevel level" "set the log4j level for this command"
  hadoop_add_option "--hosts filename" "list of hosts to use in worker mode"
  hadoop_add_option "--workers" "turn on worker mode"

  hadoop_add_subcommand "app|application" client "prints application(s) report/kill application/manage long running application"
  hadoop_add_subcommand "applicationattempt" client "prints applicationattempt(s) report"
  hadoop_add_subcommand "classpath" client "prints the class path needed to get the hadoop jar and the required libraries"
  hadoop_add_subcommand "cluster" client "prints cluster information"
  hadoop_add_subcommand "container" client "prints container(s) report"
  hadoop_add_subcommand "daemonlog" admin "get/set the log level for each daemon"
  hadoop_add_subcommand "envvars" client "display computed Hadoop environment variables"
  hadoop_add_subcommand "globalpolicygenerator" daemon "run the Global Policy Generator"
  hadoop_add_subcommand "jar <jar>" client "run a jar file"
  hadoop_add_subcommand "logs" client "dump container logs"
  hadoop_add_subcommand "node" admin "prints node report(s)"
  hadoop_add_subcommand "nodemanager" daemon "run a nodemanager on each worker"
  hadoop_add_subcommand "proxyserver" daemon "run the web app proxy server"
  hadoop_add_subcommand "queue" client "prints queue information"
  hadoop_add_subcommand "registrydns" daemon "run the registry DNS server"
  hadoop_add_subcommand "resourcemanager" daemon "run the ResourceManager"
  hadoop_add_subcommand "fs2cs" client "converts Fair Scheduler configuration to Capacity Scheduler (EXPERIMENTAL)"
  hadoop_add_subcommand "rmadmin" admin "admin tools"
  hadoop_add_subcommand "router" daemon "run the Router daemon"
  hadoop_add_subcommand "routeradmin" admin "router admin tools"
  hadoop_add_subcommand "schedulerconf" client "Updates scheduler configuration"
  hadoop_add_subcommand "scmadmin" admin "SharedCacheManager admin tools"
  hadoop_add_subcommand "sharedcachemanager" daemon "run the SharedCacheManager daemon"
  hadoop_add_subcommand "timelinereader" client "run the timeline reader server"
  hadoop_add_subcommand "timelineserver" daemon "run the timeline server"
  hadoop_add_subcommand "top" client "view cluster information"
  hadoop_add_subcommand "nodeattributes" client "node attributes cli client"
  hadoop_add_subcommand "version" client "print the version"
  hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" true
}

## @description  Default command handler for yarn command
## @audience     public
## @stability    stable
## @replaceable  no
## @param        CLI arguments
function yarncmd_case
{
  subcmd=$1
  shift

  case ${subcmd} in
    app|application|applicationattempt|container)
      HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.ApplicationCLI
      set -- "${subcmd}" "$@"
      HADOOP_SUBCMD_ARGS=("$@")
      local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\
${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
${HADOOP_HDFS_HOME}/${HDFS_DIR},\
${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
      hadoop_translate_cygwin_path sld
      hadoop_add_param HADOOP_OPTS service.libdir "-Dservice.libdir=${sld}"
    ;;
    classpath)
      hadoop_do_classpath_subcommand HADOOP_CLASSNAME "$@"
    ;;
    cluster)
      HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.ClusterCLI
    ;;
    daemonlog)
      HADOOP_CLASSNAME=org.apache.hadoop.log.LogLevel
    ;;
    envvars)
      echo "JAVA_HOME='${JAVA_HOME}'"
      echo "HADOOP_YARN_HOME='${HADOOP_YARN_HOME}'"
      echo "YARN_DIR='${YARN_DIR}'"
      echo "YARN_LIB_JARS_DIR='${YARN_LIB_JARS_DIR}'"
      echo "HADOOP_CONF_DIR='${HADOOP_CONF_DIR}'"
      echo "HADOOP_TOOLS_HOME='${HADOOP_TOOLS_HOME}'"
      echo "HADOOP_TOOLS_DIR='${HADOOP_TOOLS_DIR}'"
      echo "HADOOP_TOOLS_LIB_JARS_DIR='${HADOOP_TOOLS_LIB_JARS_DIR}'"
      exit 0
    ;;
    globalpolicygenerator)
      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
      HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.globalpolicygenerator.GlobalPolicyGenerator'
      # Backwards compatibility
      if [[ -n "${YARN_GLOBALPOLICYGENERATOR_HEAPSIZE}" ]]; then
        HADOOP_HEAPSIZE_MAX="${YARN_GLOBALPOLICYGENERATOR_HEAPSIZE}"
      fi
    ;;
    jar)
      HADOOP_CLASSNAME=org.apache.hadoop.util.RunJar
    ;;
    historyserver)
      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
      echo "DEPRECATED: Use of this command to start the timeline server is deprecated." 1>&2
      echo "Instead use the timelineserver command for it." 1>&2
      echo "Starting the History Server anyway..." 1>&2
      HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'
    ;;
    logs)
      HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.LogsCLI
    ;;
    node)
      HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.NodeCLI
    ;;
    nodemanager)
      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
      hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/*"
      hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*" before
      HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.nodemanager.NodeManager'
      # Backwards compatibility
      if [[ -n "${YARN_NODEMANAGER_HEAPSIZE}" ]]; then
        HADOOP_HEAPSIZE_MAX="${YARN_NODEMANAGER_HEAPSIZE}"
      fi
    ;;
    proxyserver)
      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
      HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer'
      # Backwards compatibility
      if [[ -n "${YARN_PROXYSERVER_HEAPSIZE}" ]]; then
        HADOOP_HEAPSIZE_MAX="${YARN_PROXYSERVER_HEAPSIZE}"
      fi
    ;;
    queue)
      HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.QueueCLI
    ;;
    registrydns)
      echo "DEPRECATED: Use of this command is deprecated." 1>&2
      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
      HADOOP_SECURE_CLASSNAME='org.apache.hadoop.registry.server.dns.PrivilegedRegistryDNSStarter'
      HADOOP_CLASSNAME='org.apache.hadoop.registry.server.dns.RegistryDNSServer'
    ;;
    resourcemanager)
      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
      hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/*"
      hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*" before
      HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager'
      # Backwards compatibility
      if [[ -n "${YARN_RESOURCEMANAGER_HEAPSIZE}" ]]; then
        HADOOP_HEAPSIZE_MAX="${YARN_RESOURCEMANAGER_HEAPSIZE}"
      fi
      local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\
${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
${HADOOP_HDFS_HOME}/${HDFS_DIR},\
${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
      hadoop_translate_cygwin_path sld
      hadoop_add_param HADOOP_OPTS service.libdir "-Dservice.libdir=${sld}"
    ;;
    fs2cs)
      HADOOP_CLASSNAME="org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigToCSConfigConverterMain"
    ;;
    rmadmin)
      HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.cli.RMAdminCLI'
    ;;
    router)
      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
      HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.router.Router'
      # Backwards compatibility
      if [[ -n "${YARN_ROUTER_HEAPSIZE}" ]]; then
        HADOOP_HEAPSIZE_MAX="${YARN_ROUTER_HEAPSIZE}"
      fi
    ;;
    routeradmin)
      HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.cli.RouterCLI'
    ;;
    schedulerconf)
    HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.cli.SchedConfCLI'
    ;;
    scmadmin)
      HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.SCMAdmin'
    ;;
    sharedcachemanager)
      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
      HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager'
    ;;
    timelinereader)
      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
      hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/*"
      hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*" before
      HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer'
    ;;
	nodeattributes)
      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="false"
      HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.cli.NodeAttributesCLI'
	;;
    timelineserver)
      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
      HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'
      # Backwards compatibility
      if [[ -n "${YARN_TIMELINESERVER_HEAPSIZE}" ]]; then
        HADOOP_HEAPSIZE_MAX="${YARN_TIMELINESERVER_HEAPSIZE}"
      fi
    ;;
    version)
      HADOOP_CLASSNAME=org.apache.hadoop.util.VersionInfo
    ;;
    top)
      doNotSetCols=0
      doNotSetRows=0
      for i in "$@"; do
        if [[ $i == "-cols" ]]; then
          doNotSetCols=1
        fi
        if [[ $i == "-rows" ]]; then
          doNotSetRows=1
        fi
      done
      if [ $doNotSetCols == 0 ] && [ -n "${TERM}" ]; then
        cols=$(tput cols)
        if [ -n "$cols" ]; then
          args=( $@ )
          args=("${args[@]}" "-cols" "$cols")
          set -- "${args[@]}"
        fi
      fi
      if [ $doNotSetRows == 0 ] && [ -n "${TERM}" ]; then
        rows=$(tput lines)
        if [ -n "$rows" ]; then
          args=( $@ )
          args=("${args[@]}" "-rows" "$rows")
          set -- "${args[@]}"
        fi
      fi
      HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.TopCLI
      HADOOP_SUBCMD_ARGS=("$@")
    ;;
    *)
      HADOOP_CLASSNAME="${subcmd}"
      if ! hadoop_validate_classname "${HADOOP_CLASSNAME}"; then
        hadoop_exit_with_usage 1
      fi
    ;;
  esac
}

# let's locate libexec...
if [[ -n "${HADOOP_HOME}" ]]; then
  HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
else
  bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P)
  HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
fi

HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}"
HADOOP_NEW_CONFIG=true
if [[ -f "${HADOOP_LIBEXEC_DIR}/yarn-config.sh" ]]; then
  # shellcheck source=./hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh
  . "${HADOOP_LIBEXEC_DIR}/yarn-config.sh"
else
  echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/yarn-config.sh." 2>&1
  exit 1
fi

# now that we have support code, let's abs MYNAME so we can use it later
MYNAME=$(hadoop_abs "${MYNAME}")

# if no args specified, show usage
if [[ $# = 0 ]]; then
  hadoop_exit_with_usage 1
fi

# get arguments
HADOOP_SUBCMD=$1
shift

if hadoop_need_reexec yarn "${HADOOP_SUBCMD}"; then
  hadoop_uservar_su yarn "${HADOOP_SUBCMD}" \
    "${MYNAME}" \
    "--reexec" \
    "${HADOOP_USER_PARAMS[@]}"
  exit $?
fi

hadoop_verify_user_perm "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}"

HADOOP_SUBCMD_ARGS=("$@")

if declare -f yarn_subcommand_"${HADOOP_SUBCMD}" >/dev/null 2>&1; then
  hadoop_debug "Calling dynamically: yarn_subcommand_${HADOOP_SUBCMD} ${HADOOP_SUBCMD_ARGS[*]}"
  "yarn_subcommand_${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}"
else
  yarncmd_case "${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}"
fi

# It's unclear if YARN_CLIENT_OPTS is actually a useful
# thing to have separate from HADOOP_CLIENT_OPTS.  Someone
# might use it, so let's not deprecate it and just override
# HADOOP_CLIENT_OPTS instead before we (potentially) add it
# to the command line
if [[ -n "${YARN_CLIENT_OPTS}" ]]; then
  HADOOP_CLIENT_OPTS=${YARN_CLIENT_OPTS}
fi

hadoop_add_client_opts

if [[ ${HADOOP_WORKER_MODE} = true ]]; then
  hadoop_common_worker_mode_execute "${HADOOP_YARN_HOME}/bin/yarn" "${HADOOP_USER_PARAMS[@]}"
  exit $?
fi

hadoop_subcommand_opts "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}"

# everything is in globals at this point, so call the generic handler
hadoop_generic_java_subcmd_handler