#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # The Hadoop command script # # Environment Variables # # JAVA_HOME The java implementation to use. Overrides JAVA_HOME. # # YARN_CLASSPATH Extra Java CLASSPATH entries. # # YARN_HEAPSIZE The maximum amount of heap to use, in MB. # Default is 1000. # # YARN_OPTS Extra Java runtime options. # # YARN_NAMENODE_OPTS These options are added to YARN_OPTS # YARN_CLIENT_OPTS when the respective command is run. # YARN_{COMMAND}_OPTS etc YARN_JT_OPTS applies to JobTracker # for e.g. YARN_CLIENT_OPTS applies to # more than one command (fs, dfs, fsck, # dfsadmin etc) # # YARN_CONF_DIR Alternate conf dir. Default is ${YARN_HOME}/conf. # # YARN_ROOT_LOGGER The root appender. Default is INFO,console # bin=`dirname "${BASH_SOURCE-$0}"` bin=`cd "$bin"; pwd` DEFAULT_LIBEXEC_DIR="$bin" HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} . $HADOOP_LIBEXEC_DIR/yarn-config.sh cygwin=false case "`uname`" in CYGWIN*) cygwin=true;; esac # if no args specified, show usage if [ $# = 0 ]; then echo "Usage: yarn [--config confdir] COMMAND" echo "where COMMAND is one of:" echo " resourcemanager run the ResourceManager" echo " nodemanager run a nodemanager on each slave" echo " historyserver run job history servers as a standalone daemon" echo " rmadmin admin tools" echo " version print the version" echo " jar run a jar file" echo " logs dump container logs" echo " classpath prints the class path needed to get the" echo " Hadoop jar and the required libraries" echo " daemonlog get/set the log level for each daemon" echo " or" echo " CLASSNAME run the class named CLASSNAME" echo "Most commands print help when invoked w/o parameters." exit 1 fi # get arguments COMMAND=$1 shift if [ -f "${YARN_CONF_DIR}/yarn-env.sh" ]; then . "${YARN_CONF_DIR}/yarn-env.sh" fi # some Java parameters if [ "$JAVA_HOME" != "" ]; then #echo "run java in $JAVA_HOME" JAVA_HOME=$JAVA_HOME fi if [ "$JAVA_HOME" = "" ]; then echo "Error: JAVA_HOME is not set." exit 1 fi JAVA=$JAVA_HOME/bin/java JAVA_HEAP_MAX=-Xmx1000m # check envvars which might override default args if [ "$YARN_HEAPSIZE" != "" ]; then #echo "run with heapsize $YARN_HEAPSIZE" JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m" #echo $JAVA_HEAP_MAX fi # CLASSPATH initially contains $HADOOP_CONF_DIR & $YARN_CONF_DIR if [ ! -d "$HADOOP_CONF_DIR" ]; then echo No HADOOP_CONF_DIR set. echo Please specify it either in yarn-env.sh or in the environment. exit 1 fi CLASSPATH="${HADOOP_CONF_DIR}:${YARN_CONF_DIR}" CLASSPATH=${CLASSPATH}:${YARN_CLASSPATH} CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar # for developers, add Hadoop classes to CLASSPATH if [ -d "$YARN_HOME/yarn-api/target/classes" ]; then CLASSPATH=${CLASSPATH}:$YARN_HOME/yarn-api/target/classes fi if [ -d "$YARN_HOME/yarn-common/target/classes" ]; then CLASSPATH=${CLASSPATH}:$YARN_HOME/yarn-common/target/classes fi if [ -d "$YARN_HOME/yarn-mapreduce/target/classes" ]; then CLASSPATH=${CLASSPATH}:$YARN_HOME/yarn-mapreduce/target/classes fi if [ -d "$YARN_HOME/yarn-master-worker/target/classes" ]; then CLASSPATH=${CLASSPATH}:$YARN_HOME/yarn-master-worker/target/classes fi if [ -d "$YARN_HOME/yarn-server/yarn-server-nodemanager/target/classes" ]; then CLASSPATH=${CLASSPATH}:$YARN_HOME/yarn-server/yarn-server-nodemanager/target/classes fi if [ -d "$YARN_HOME/yarn-server/yarn-server-common/target/classes" ]; then CLASSPATH=${CLASSPATH}:$YARN_HOME/yarn-server/yarn-server-common/target/classes fi if [ -d "$YARN_HOME/yarn-server/yarn-server-resourcemanager/target/classes" ]; then CLASSPATH=${CLASSPATH}:$YARN_HOME/yarn-server/yarn-server-resourcemanager/target/classes fi if [ -d "$YARN_HOME/build/test/classes" ]; then CLASSPATH=${CLASSPATH}:$YARN_HOME/target/test/classes fi if [ -d "$YARN_HOME/build/tools" ]; then CLASSPATH=${CLASSPATH}:$YARN_HOME/build/tools fi # so that filenames w/ spaces are handled correctly in loops below IFS= # add hadoop-common libs to CLASSPATH if [ ! -d "$HADOOP_COMMON_HOME" ]; then echo No HADOOP_COMMON_HOME set. echo Please specify it either in yarn-env.sh or in the environment. exit 1 fi CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/share/hadoop/common'/*' CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/share/hadoop/common/lib'/*' # add hadoop-hdfs libs to CLASSPATH if [ ! -d "$HADOOP_HDFS_HOME" ]; then echo No HADOOP_HDFS_HOME set. echo Please specify it either in yarn-env.sh or in the environment. exit 1 fi CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/share/hadoop/hdfs'/*' CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib'/*' # add yarn libs to CLASSPATH CLASSPATH=${CLASSPATH}:$YARN_HOME/modules'/*' CLASSPATH=${CLASSPATH}:$YARN_HOME/lib'/*' # default log directory & file if [ "$YARN_LOG_DIR" = "" ]; then YARN_LOG_DIR="$YARN_HOME/logs" fi if [ "$YARN_LOGFILE" = "" ]; then YARN_LOGFILE='yarn.log' fi YARN_JOB_HISTORYSERVER_OPTS="-Dmapred.jobsummary.logger=${YARN_JHS_LOGGER:-INFO,console}" # restore ordinary behaviour unset IFS # figure out which class to run if [ "$COMMAND" = "classpath" ] ; then if $cygwin; then CLASSPATH=`cygpath -p -w "$CLASSPATH"` fi echo $CLASSPATH exit elif [ "$COMMAND" = "rmadmin" ] ; then CLASS='org.apache.hadoop.yarn.server.resourcemanager.tools.RMAdmin' YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS" elif [ "$COMMAND" = "resourcemanager" ] ; then CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/rm-config/log4j.properties CLASS='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager' YARN_OPTS="$YARN_OPTS $YARN_RESOURCEMANAGER_OPTS" elif [ "$COMMAND" = "nodemanager" ] ; then CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/nm-config/log4j.properties CLASS='org.apache.hadoop.yarn.server.nodemanager.NodeManager' if [[ $EUID -eq 0 ]]; then YARN_OPTS="$YARN_OPTS -jvm server $YARN_NODEMANAGER_OPTS" else YARN_OPTS="$YARN_OPTS -server $YARN_NODEMANAGER_OPTS" fi elif [ "$COMMAND" = "historyserver" ] ; then CLASS=org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer YARN_OPTS="$YARN_OPTS $YARN_JOB_HISTORYSERVER_OPTS" elif [ "$COMMAND" = "proxyserver" ] ; then CLASS='org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer' YARN_OPTS="$YARN_OPTS $YARN_PROXYSERVER_OPTS" elif [ "$COMMAND" = "version" ] ; then CLASS=org.apache.hadoop.util.VersionInfo YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS" elif [ "$COMMAND" = "jar" ] ; then CLASS=org.apache.hadoop.util.RunJar YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS" elif [ "$COMMAND" = "logs" ] ; then CLASS=org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.LogDumper YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS" elif [ "$COMMAND" = "daemonlog" ] ; then CLASS=org.apache.hadoop.log.LogLevel YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS" else CLASS=$COMMAND fi # cygwin path translation if $cygwin; then CLASSPATH=`cygpath -p -w "$CLASSPATH"` YARN_HOME=`cygpath -w "$YARN_HOME"` YARN_LOG_DIR=`cygpath -w "$YARN_LOG_DIR"` TOOL_PATH=`cygpath -p -w "$TOOL_PATH"` fi # cygwin path translation if $cygwin; then JAVA_LIBRARY_PATH=`cygpath -p "$JAVA_LIBRARY_PATH"` fi YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR" YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR" YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE" YARN_OPTS="$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE" YARN_OPTS="$YARN_OPTS -Dyarn.home.dir=$YARN_HOME" YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}" YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}" if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH" fi echo "$JAVA" -Dproc_$COMMAND $JAVA_HEAP_MAX $YARN_OPTS -classpath "$CLASSPATH" $CLASS "$@" exec "$JAVA" -Dproc_$COMMAND $JAVA_HEAP_MAX $YARN_OPTS -classpath "$CLASSPATH" $CLASS "$@" fi