From 0abb973f0998ab70e5d3597fcc7ea29ca4d6b583 Mon Sep 17 00:00:00 2001 From: cnauroth Date: Wed, 12 Nov 2014 21:41:19 -0800 Subject: [PATCH] HADOOP-7984. Add hadoop --loglevel option to change log level. Contributed by Aikira AJISAKA. --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../hadoop-common/src/main/bin/hadoop | 2 +- .../hadoop-common/src/main/bin/hadoop-config.cmd | 16 +++++++++++++++- .../hadoop-common/src/main/bin/hadoop-config.sh | 6 ++++++ .../src/main/bin/hadoop-functions.sh | 5 +++-- .../hadoop-common/src/main/bin/hadoop.cmd | 10 +++++++++- .../src/site/apt/CommandsManual.apt.vm | 7 ++++++- .../hadoop-hdfs/src/main/bin/hdfs | 3 ++- .../hadoop-hdfs/src/main/bin/hdfs.cmd | 10 +++++++++- .../hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm | 8 ++++++-- hadoop-mapreduce-project/bin/mapred | 3 ++- hadoop-mapreduce-project/bin/mapred-config.sh | 2 +- hadoop-mapreduce-project/bin/mapred.cmd | 11 ++++++++++- hadoop-mapreduce-project/conf/mapred-env.cmd | 2 +- .../src/site/apt/MapredCommands.apt.vm | 5 ++++- hadoop-yarn-project/hadoop-yarn/bin/yarn | 3 ++- .../hadoop-yarn/bin/yarn-config.sh | 2 +- hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 10 +++++++++- .../hadoop-yarn/conf/yarn-env.cmd | 2 +- .../src/site/apt/YarnCommands.apt.vm | 5 ++++- 20 files changed, 95 insertions(+), 20 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index f8fca1645a..1a304d70eb 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -357,6 +357,9 @@ Release 2.7.0 - UNRELEASED HADOOP-10987. Provide an iterator-based listing API for FileSystem (kihwal) + HADOOP-7984. Add hadoop --loglevel option to change log level. + (Akira AJISAKA via cnauroth) + IMPROVEMENTS HADOOP-11156. DelegateToFileSystem should implement diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop b/hadoop-common-project/hadoop-common/src/main/bin/hadoop index 2a065c211e..eccd322351 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop @@ -17,7 +17,7 @@ function hadoop_usage() { - echo "Usage: hadoop [--config confdir] COMMAND" + echo "Usage: hadoop [--config confdir] [--loglevel loglevel] COMMAND" echo " where COMMAND is one of:" echo " archive -archiveName NAME -p * " echo " create a Hadoop archive" diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.cmd b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.cmd index d8da5b16aa..8ae70fbb34 100644 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.cmd +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.cmd @@ -88,6 +88,16 @@ if "%1" == "--hosts" ( shift ) +@rem +@rem Set log level. Default to INFO. +@rem + +if "%1" == "--loglevel" ( + set HADOOP_LOGLEVEL=%2 + shift + shift +) + if exist %HADOOP_CONF_DIR%\hadoop-env.cmd ( call %HADOOP_CONF_DIR%\hadoop-env.cmd ) @@ -157,8 +167,12 @@ if not defined HADOOP_LOGFILE ( set HADOOP_LOGFILE=hadoop.log ) +if not defined HADOOP_LOGLEVEL ( + set HADOOP_LOGLEVEL=INFO +) + if not defined HADOOP_ROOT_LOGGER ( - set HADOOP_ROOT_LOGGER=INFO,console + set HADOOP_ROOT_LOGGER=%HADOOP_LOGLEVEL%,console ) @rem diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh index 81a5867e17..89b0c932b3 100644 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh @@ -148,6 +148,12 @@ while [[ -z "${_hadoop_common_done}" ]]; do hadoop_populate_slaves_file "$1" shift ;; + --loglevel) + shift + # shellcheck disable=SC2034 + HADOOP_LOGLEVEL="$1" + shift + ;; *) _hadoop_common_done=true ;; diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh index 52d5af9ceb..4f75a7ccf5 100644 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh @@ -155,11 +155,12 @@ function hadoop_basic_init HADOOP_IDENT_STRING=${HADOP_IDENT_STRING:-$USER} HADOOP_LOG_DIR=${HADOOP_LOG_DIR:-"${HADOOP_PREFIX}/logs"} HADOOP_LOGFILE=${HADOOP_LOGFILE:-hadoop.log} + HADOOP_LOGLEVEL=${HADOOP_LOGLEVEL:-INFO} HADOOP_NICENESS=${HADOOP_NICENESS:-0} HADOOP_STOP_TIMEOUT=${HADOOP_STOP_TIMEOUT:-5} HADOOP_PID_DIR=${HADOOP_PID_DIR:-/tmp} - HADOOP_ROOT_LOGGER=${HADOOP_ROOT_LOGGER:-INFO,console} - HADOOP_DAEMON_ROOT_LOGGER=${HADOOP_DAEMON_ROOT_LOGGER:-INFO,RFA} + HADOOP_ROOT_LOGGER=${HADOOP_ROOT_LOGGER:-${HADOOP_LOGLEVEL},console} + HADOOP_DAEMON_ROOT_LOGGER=${HADOOP_DAEMON_ROOT_LOGGER:-${HADOOP_LOGLEVEL},RFA} HADOOP_SECURITY_LOGGER=${HADOOP_SECURITY_LOGGER:-INFO,NullAppender} HADOOP_HEAPSIZE=${HADOOP_HEAPSIZE:-1024} HADOOP_SSH_OPTS=${HADOOP_SSH_OPTS:-"-o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=10s"} diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd b/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd index 2e3e86fea0..fc2e9b743a 100644 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd @@ -88,6 +88,10 @@ call :updatepath %HADOOP_BIN_PATH% shift shift ) + if "%1" == "--loglevel" ( + shift + shift + ) set hadoop-command=%1 if not defined hadoop-command ( @@ -238,6 +242,10 @@ call :updatepath %HADOOP_BIN_PATH% shift shift ) + if "%1" == "--loglevel" ( + shift + shift + ) if [%2] == [] goto :eof shift set _arguments= @@ -256,7 +264,7 @@ call :updatepath %HADOOP_BIN_PATH% goto :eof :print_usage - @echo Usage: hadoop [--config confdir] COMMAND + @echo Usage: hadoop [--config confdir] [--loglevel loglevel] COMMAND @echo where COMMAND is one of: @echo fs run a generic filesystem user client @echo version print the version diff --git a/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm index 1608691097..7771aa13ff 100644 --- a/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm +++ b/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm @@ -27,7 +27,8 @@ Overview hadoop script without any arguments prints the description for all commands. - Usage: <<>> + Usage: <<>> Hadoop has an option parsing framework that employs parsing generic options as well as running classes. @@ -37,6 +38,10 @@ Overview *-----------------------+---------------+ | <<<--config confdir>>>| Overwrites the default Configuration directory. Default is <<<${HADOOP_HOME}/conf>>>. *-----------------------+---------------+ +| <<<--loglevel loglevel>>>| Overwrites the log level. Valid log levels are +| | FATAL, ERROR, WARN, INFO, DEBUG, and TRACE. +| | Default is INFO. +*-----------------------+---------------+ | GENERIC_OPTIONS | The common set of options supported by multiple commands. | COMMAND_OPTIONS | Various commands with their options are described in the following sections. The commands have been grouped into User Commands and Administration Commands. *-----------------------+---------------+ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs index d47df118a0..36f2e7f51e 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs @@ -17,7 +17,8 @@ function hadoop_usage { - echo "Usage: hdfs [--config confdir] [--daemon (start|stop|status)] COMMAND" + echo "Usage: hdfs [--config confdir] [--daemon (start|stop|status)]" + echo " [--loglevel loglevel] COMMAND" echo " where COMMAND is one of:" echo " balancer run a cluster balancing utility" echo " cacheadmin configure the HDFS cache" diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd index 69424ed745..00ecfe2795 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd @@ -34,6 +34,10 @@ if "%1" == "--config" ( shift shift ) +if "%1" == "--loglevel" ( + shift + shift +) :main if exist %HADOOP_CONF_DIR%\hadoop-env.cmd ( @@ -165,6 +169,10 @@ goto :eof shift shift ) + if "%1" == "--loglevel" ( + shift + shift + ) if [%2] == [] goto :eof shift set _hdfsarguments= @@ -183,7 +191,7 @@ goto :eof goto :eof :print_usage - @echo Usage: hdfs [--config confdir] COMMAND + @echo Usage: hdfs [--config confdir] [--loglevel loglevel] COMMAND @echo where COMMAND is one of: @echo dfs run a filesystem command on the file systems supported in Hadoop. @echo namenode -format format the DFS filesystem diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm index 36abea7e01..281221dab0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm @@ -26,8 +26,8 @@ HDFS Commands Guide hdfs script without any arguments prints the description for all commands. - Usage: <<>> + Usage: <<>> Hadoop has an option parsing framework that employs parsing generic options as well as running classes. @@ -38,6 +38,10 @@ HDFS Commands Guide | <<<--config confdir>>>| Overwrites the default Configuration directory. | | Default is <<<${HADOOP_HOME}/conf>>>. *-----------------------+---------------+ +| <<<--loglevel loglevel>>>| Overwrites the log level. Valid log levels are +| | FATAL, ERROR, WARN, INFO, DEBUG, and TRACE. +| | Default is INFO. +*-----------------------+---------------+ | GENERIC_OPTIONS | The common set of options supported by multiple | | commands. Full list is | | {{{../hadoop-common/CommandsManual.html#Generic_Options}here}}. diff --git a/hadoop-mapreduce-project/bin/mapred b/hadoop-mapreduce-project/bin/mapred index 2163cada29..1fa5d443d2 100755 --- a/hadoop-mapreduce-project/bin/mapred +++ b/hadoop-mapreduce-project/bin/mapred @@ -17,7 +17,8 @@ function hadoop_usage { - echo "Usage: mapred [--config confdir] [--daemon (start|stop|status)] COMMAND" + echo "Usage: mapred [--config confdir] [--daemon (start|stop|status)]" + echo " [--loglevel loglevel] COMMAND" echo " where COMMAND is one of:" echo " archive -archiveName NAME -p * create a hadoop archive" diff --git a/hadoop-mapreduce-project/bin/mapred-config.sh b/hadoop-mapreduce-project/bin/mapred-config.sh index cc62b9e59c..39baf4eadf 100644 --- a/hadoop-mapreduce-project/bin/mapred-config.sh +++ b/hadoop-mapreduce-project/bin/mapred-config.sh @@ -49,7 +49,7 @@ function hadoop_subproject_init HADOOP_PID_DIR="${HADOOP_MAPRED_PID_DIR:-$HADOOP_PID_DIR}" HADOOP_MAPRED_PID_DIR="${HADOOP_PID_DIR}" - HADOOP_ROOT_LOGGER="${HADOOP_MAPRED_ROOT_LOGGER:-INFO,console}" + HADOOP_ROOT_LOGGER="${HADOOP_MAPRED_ROOT_LOGGER:-${HADOOP_LOGLEVEL},console}" HADOOP_MAPRED_ROOT_LOGGER="${HADOOP_ROOT_LOGGER}" HADOOP_MAPRED_HOME="${HADOOP_MAPRED_HOME:-$HADOOP_PREFIX}" diff --git a/hadoop-mapreduce-project/bin/mapred.cmd b/hadoop-mapreduce-project/bin/mapred.cmd index 8457ba073c..bb59c03085 100644 --- a/hadoop-mapreduce-project/bin/mapred.cmd +++ b/hadoop-mapreduce-project/bin/mapred.cmd @@ -37,6 +37,11 @@ if "%1" == "--config" ( shift ) +if "%1" == "--loglevel" ( + shift + shift +) + :main if exist %MAPRED_CONF_DIR%\mapred-env.cmd ( call %MAPRED_CONF_DIR%\mapred-env.cmd @@ -162,6 +167,10 @@ goto :eof shift shift ) + if "%1" == "--loglevel" ( + shift + shift + ) shift set _mapredarguments= :MakeCmdArgsLoop @@ -184,7 +193,7 @@ goto :eof goto print_usage :print_usage - @echo Usage: mapred [--config confdir] COMMAND + @echo Usage: mapred [--config confdir] [--loglevel loglevel] COMMAND @echo where COMMAND is one of: @echo job manipulate MapReduce jobs @echo queue get information regarding JobQueues diff --git a/hadoop-mapreduce-project/conf/mapred-env.cmd b/hadoop-mapreduce-project/conf/mapred-env.cmd index 610d593b93..f27943f44a 100644 --- a/hadoop-mapreduce-project/conf/mapred-env.cmd +++ b/hadoop-mapreduce-project/conf/mapred-env.cmd @@ -16,5 +16,5 @@ set HADOOP_JOB_HISTORYSERVER_HEAPSIZE=1000 -set HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA +set HADOOP_MAPRED_ROOT_LOGGER=%HADOOP_LOGLEVEL%,RFA diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapredCommands.apt.vm b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapredCommands.apt.vm index 9edc41d13d..e011563d3f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapredCommands.apt.vm +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapredCommands.apt.vm @@ -28,7 +28,7 @@ MapReduce Commands Guide MapReduce commands are invoked by the <<>> script. Running the script without any arguments prints the description for all commands. - Usage: <<>> + Usage: <<>> MapReduce has an option parsing framework that employs parsing generic options as well as running classes. @@ -39,6 +39,9 @@ MapReduce Commands Guide | --config confdir | Overwrites the default Configuration directory. Default | | is $\{HADOOP_PREFIX\}/conf. *-------------------------+---------------------------------------------------+ +| --loglevel loglevel | Overwrites the log level. Valid log levels are FATAL, +| | ERROR, WARN, INFO, DEBUG, and TRACE. Default is INFO. +*-------------------------+---------------------------------------------------+ | COMMAND COMMAND_OPTIONS | Various commands with their options are described | | in the following sections. The commands have been | | grouped into {{User Commands}} and diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index 0d83674118..90e5527a3a 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -17,7 +17,8 @@ function hadoop_usage { - echo "Usage: yarn [--config confdir] [--daemon (start|stop|status)] COMMAND" + echo "Usage: yarn [--config confdir] [--daemon (start|stop|status)]" + echo " [--loglevel loglevel] COMMAND" echo "where COMMAND is one of:" echo " application prints application(s) report/kill application" echo " applicationattempt prints applicationattempt(s) report" diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh b/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh index 372e8efc36..7fd7eada88 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh @@ -58,7 +58,7 @@ function hadoop_subproject_init HADOOP_PID_DIR="${YARN_PID_DIR:-$HADOOP_PID_DIR}" YARN_PID_DIR="${HADOOP_PID_DIR}" - HADOOP_ROOT_LOGGER="${YARN_ROOT_LOGGER:-INFO,console}" + HADOOP_ROOT_LOGGER="${YARN_ROOT_LOGGER:-${HADOOP_LOGLEVEL},console}" YARN_ROOT_LOGGER="${HADOOP_ROOT_LOGGER}" HADOOP_YARN_HOME="${HADOOP_YARN_HOME:-$HADOOP_PREFIX}" diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd index 7b7fdc4db4..b3286e23ce 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd @@ -64,6 +64,10 @@ if "%1" == "--config" ( shift shift ) +if "%1" == "--loglevel" ( + shift + shift +) :main if exist %YARN_CONF_DIR%\yarn-env.cmd ( @@ -273,6 +277,10 @@ goto :eof shift shift ) + if "%1" == "--loglevel" ( + shift + shift + ) if [%2] == [] goto :eof shift set _yarnarguments= @@ -291,7 +299,7 @@ goto :eof goto :eof :print_usage - @echo Usage: yarn [--config confdir] COMMAND + @echo Usage: yarn [--config confdir] [--loglevel loglevel] COMMAND @echo where COMMAND is one of: @echo resourcemanager run the ResourceManager @echo nodemanager run a nodemanager on each slave diff --git a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.cmd b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.cmd index 3329f8fdc3..d863c1e375 100644 --- a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.cmd +++ b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.cmd @@ -42,7 +42,7 @@ if not defined YARN_POLICYFILE ( ) if not defined YARN_ROOT_LOGGER ( - set YARN_ROOT_LOGGER=INFO,console + set YARN_ROOT_LOGGER=%HADOOP_LOGLEVEL%,console ) set YARN_OPTS=%YARN_OPTS% -Dhadoop.log.dir=%YARN_LOG_DIR% diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm index d91c513348..cf41271af1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm @@ -26,7 +26,7 @@ Yarn Commands without any arguments prints the description for all commands. ------ -Usage: yarn [--config confdir] COMMAND +Usage: yarn [--config confdir] [--loglevel loglevel] COMMAND ------ Yarn has an option parsing framework that employs parsing generic options as @@ -38,6 +38,9 @@ Usage: yarn [--config confdir] COMMAND | --config confdir | Overwrites the default Configuration directory. Default | | is $\{HADOOP_PREFIX\}/conf. *---------------+--------------+ +| --loglevel loglevel | Overwrites the log level. Valid log levels are FATAL, +| | ERROR, WARN, INFO, DEBUG, and TRACE. Default is INFO. +*---------------+--------------+ | COMMAND COMMAND_OPTIONS | Various commands with their options are described | | in the following sections. The commands have been | | grouped into {{User Commands}} and