diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 793e583217..d7ebeac0e2 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -141,6 +141,8 @@ Trunk (Unreleased) HADOOP-11352 Clean up test-patch.sh to disable "+1 contrib tests" (Akira AJISAKA via stevel) + HADOOP-10788. Rewrite kms to use new shell framework (John Smith via aw) + BUG FIXES HADOOP-9451. Fault single-layer config if node group topology is enabled. diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh index a953e0c7cf..799aad0e1b 100644 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh @@ -666,6 +666,25 @@ function hadoop_finalize_classpath hadoop_add_to_classpath_userpath } +function hadoop_finalize_catalina_opts +{ + + local prefix=${HADOOP_CATALINA_PREFIX} + + hadoop_add_param CATALINA_OPTS hadoop.home.dir "-Dhadoop.home.dir=${HADOOP_PREFIX}" + if [[ -n "${JAVA_LIBRARY_PATH}" ]]; then + hadoop_add_param CATALINA_OPTS java.library.path "-Djava.library.path=${JAVA_LIBRARY_PATH}" + fi + hadoop_add_param CATALINA_OPTS "${prefix}.home.dir" "-D${prefix}.home.dir=${HADOOP_PREFIX}" + hadoop_add_param CATALINA_OPTS "${prefix}.config.dir" "-D${prefix}.config.dir=${HADOOP_CATALINA_CONFIG}" + hadoop_add_param CATALINA_OPTS "${prefix}.log.dir" "-D${prefix}.log.dir=${HADOOP_CATALINA_LOG}" + hadoop_add_param CATALINA_OPTS "${prefix}.temp.dir" "-D${prefix}.temp.dir=${HADOOP_CATALINA_TEMP}" + hadoop_add_param CATALINA_OPTS "${prefix}.admin.port" "-D${prefix}.admin.port=${HADOOP_CATALINA_ADMIN_PORT}" + hadoop_add_param CATALINA_OPTS "${prefix}.http.port" "-D${prefix}.http.port=${HADOOP_CATALINA_HTTP_PORT}" + hadoop_add_param CATALINA_OPTS "${prefix}.max.threads" "-D${prefix}.max.threads=${HADOOP_CATALINA_MAX_THREADS}" + hadoop_add_param CATALINA_OPTS "${prefix}.ssl.keystore.file" "-D${prefix}.ssl.keystore.file=${HADOOP_CATALINA_SSL_KEYSTORE_FILE}" +} + function hadoop_finalize { # user classpath gets added at the last minute. this allows diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh b/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh index 44dfe6afcd..7045379114 100644 --- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh +++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh @@ -14,19 +14,17 @@ # # Set kms specific environment variables here. - -# Settings for the Embedded Tomcat that runs KMS -# Java System properties for KMS should be specified in this variable # -# export CATALINA_OPTS= +# hadoop-env.sh is read prior to this file. +# # KMS logs directory # -# export KMS_LOG=${KMS_HOME}/logs +# export KMS_LOG=${HADOOP_LOG_DIR} # KMS temporary directory # -# export KMS_TEMP=${KMS_HOME}/temp +# export KMS_TEMP=${HADOOP_PREFIX}/temp # The HTTP port used by KMS # @@ -34,7 +32,7 @@ # The Admin port used by KMS # -# export KMS_ADMIN_PORT=`expr ${KMS_HTTP_PORT} + 1` +# export KMS_ADMIN_PORT=$((KMS_HTTP_PORT + 1)) # The maximum number of Tomcat handler threads # @@ -44,12 +42,37 @@ # # export KMS_SSL_KEYSTORE_FILE=${HOME}/.keystore +# # The password of the SSL keystore if using SSL # # export KMS_SSL_KEYSTORE_PASS=password -# The full path to any native libraries that need to be loaded -# (For eg. location of natively compiled tomcat Apache portable -# runtime (APR) libraries # -# export JAVA_LIBRARY_PATH=${HOME}/lib/native +# The password of the truststore +# +# export KMS_SSL_TRUSTSTORE_PASS= + + +## +## Tomcat specific settings +## +# +# Location of tomcat +# +# export KMS_CATALINA_HOME=${HADOOP_PREFIX}/share/hadoop/kms/tomcat + +# Java System properties for KMS should be specified in this variable. +# The java.library.path and hadoop.home.dir properties are automatically +# configured. In order to supplement java.library.path, +# one should add to the JAVA_LIBRARY_PATH env var. +# +# export CATALINA_OPTS= + +# PID file +# +# export CATALINA_PID=${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-kms.pid + +# Output file +# +# export CATALINA_OUT=${KMS_LOG}/hadoop-${HADOOP_IDENT_STRING}-kms-${HOSTNAME}.out + diff --git a/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh b/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh index 3ac929a2fc..7cfb78d275 100644 --- a/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh +++ b/hadoop-common-project/hadoop-kms/src/main/libexec/kms-config.sh @@ -13,182 +13,63 @@ # limitations under the License. # -# resolve links - $0 may be a softlink -PRG="${0}" +function hadoop_subproject_init +{ + local this + local binparent + local varlist -while [ -h "${PRG}" ]; do - ls=`ls -ld "${PRG}"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "${PRG}"`/"$link" + if [[ -z "${HADOOP_KMS_ENV_PROCESSED}" ]]; then + if [[ -e "${HADOOP_CONF_DIR}/kms-env.sh" ]]; then + . "${HADOOP_CONF_DIR}/kms-env.sh" + export HADOOP_KMS_ENV_PROCESSED=true + fi fi -done -BASEDIR=`dirname ${PRG}` -BASEDIR=`cd ${BASEDIR}/..;pwd` + export HADOOP_CATALINA_PREFIX=kms + export HADOOP_CATALINA_TEMP="${KMS_TEMP:-${HADOOP_PREFIX}/temp}" -function print() { - if [ "${KMS_SILENT}" != "true" ]; then - echo "$@" + export HADOOP_CONF_DIR="${KMS_CONFIG:-${HADOOP_CONF_DIR}}" + export HADOOP_CATALINA_CONFIG="${HADOOP_CONF_DIR}" + + export HADOOP_LOG_DIR="${KMS_LOG:-${HADOOP_LOG_DIR}}" + export HADOOP_CATALINA_LOG="${HADOOP_LOG_DIR}" + + export HADOOP_CATALINA_HTTP_PORT="${KMS_HTTP_PORT:-16000}" + export HADOOP_CATALINA_ADMIN_PORT="${KMS_ADMIN_PORT:-$((HADOOP_CATALINA_HTTP_PORT+1))}" + export HADOOP_CATALINA_MAX_THREADS="${KMS_MAX_THREADS:-1000}" + + export HADOOP_CATALINA_SSL_KEYSTORE_FILE="${KMS_SSL_KEYSTORE_FILE:-${HOME}/.keystore}" + + # this is undocumented, but older versions would rip the TRUSTSTORE_PASS out of the + # CATALINA_OPTS + # shellcheck disable=SC2086 + export KMS_SSL_TRUSTSTORE_PASS=${KMS_SSL_TRUSTSTORE_PASS:-"$(echo ${CATALINA_OPTS} | grep -o 'trustStorePassword=[^ ]*' | cut -f2 -d= )"} + + export CATALINA_BASE="${CATALINA_BASE:-${HADOOP_PREFIX}/share/hadoop/kms/tomcat}" + export HADOOP_CATALINA_HOME="${KMS_CATALINA_HOME:-${CATALINA_BASE}}" + + export CATALINA_OUT="${CATALINA_OUT:-${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-kms-${HOSTNAME}.out}" + + export CATALINA_PID="${CATALINA_PID:-${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-kms.pid}" + + if [[ -n "${HADOOP_SHELL_SCRIPT_DEBUG}" ]]; then + varlist=$(env | egrep '(^KMS|^CATALINA)' | cut -f1 -d= | grep -v _PASS) + for i in ${varlist}; do + hadoop_debug "Setting ${i} to ${!i}" + done fi } -# if KMS_HOME is already set warn it will be ignored -# -if [ "${KMS_HOME}" != "" ]; then - echo "WARNING: current setting of KMS_HOME ignored" -fi - -print - -# setting KMS_HOME to the installation dir, it cannot be changed -# -export KMS_HOME=${BASEDIR} -kms_home=${KMS_HOME} -print "Setting KMS_HOME: ${KMS_HOME}" - -# if the installation has a env file, source it -# this is for native packages installations -# -if [ -e "${KMS_HOME}/bin/kms-env.sh" ]; then - print "Sourcing: ${KMS_HOME}/bin/kms-env.sh" - source ${KMS_HOME}/bin/kms-env.sh - grep "^ *export " ${KMS_HOME}/bin/kms-env.sh | sed 's/ *export/ setting/' -fi - -# verify that the sourced env file didn't change KMS_HOME -# if so, warn and revert -# -if [ "${KMS_HOME}" != "${kms_home}" ]; then - print "WARN: KMS_HOME resetting to ''${KMS_HOME}'' ignored" - export KMS_HOME=${kms_home} - print " using KMS_HOME: ${KMS_HOME}" -fi - -if [ "${KMS_CONFIG}" = "" ]; then - export KMS_CONFIG=${KMS_HOME}/etc/hadoop - print "Setting KMS_CONFIG: ${KMS_CONFIG}" +if [[ -n "${HADOOP_COMMON_HOME}" ]] && + [[ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]]; then + . "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" +elif [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then + . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" +elif [[ -e "${HADOOP_PREFIX}/libexec/hadoop-config.sh" ]]; then + . "${HADOOP_PREFIX}/libexec/hadoop-config.sh" else - print "Using KMS_CONFIG: ${KMS_CONFIG}" + echo "ERROR: Hadoop common not found." 2>&1 + exit 1 fi -kms_config=${KMS_CONFIG} - -# if the configuration dir has a env file, source it -# -if [ -e "${KMS_CONFIG}/kms-env.sh" ]; then - print "Sourcing: ${KMS_CONFIG}/kms-env.sh" - source ${KMS_CONFIG}/kms-env.sh - grep "^ *export " ${KMS_CONFIG}/kms-env.sh | sed 's/ *export/ setting/' -fi - -# verify that the sourced env file didn't change KMS_HOME -# if so, warn and revert -# -if [ "${KMS_HOME}" != "${kms_home}" ]; then - echo "WARN: KMS_HOME resetting to ''${KMS_HOME}'' ignored" - export KMS_HOME=${kms_home} -fi - -# verify that the sourced env file didn't change KMS_CONFIG -# if so, warn and revert -# -if [ "${KMS_CONFIG}" != "${kms_config}" ]; then - echo "WARN: KMS_CONFIG resetting to ''${KMS_CONFIG}'' ignored" - export KMS_CONFIG=${kms_config} -fi - -if [ "${KMS_LOG}" = "" ]; then - export KMS_LOG=${KMS_HOME}/logs - print "Setting KMS_LOG: ${KMS_LOG}" -else - print "Using KMS_LOG: ${KMS_LOG}" -fi - -if [ ! -f ${KMS_LOG} ]; then - mkdir -p ${KMS_LOG} -fi - -if [ "${KMS_TEMP}" = "" ]; then - export KMS_TEMP=${KMS_HOME}/temp - print "Setting KMS_TEMP: ${KMS_TEMP}" -else - print "Using KMS_TEMP: ${KMS_TEMP}" -fi - -if [ ! -f ${KMS_TEMP} ]; then - mkdir -p ${KMS_TEMP} -fi - -if [ "${KMS_HTTP_PORT}" = "" ]; then - export KMS_HTTP_PORT=16000 - print "Setting KMS_HTTP_PORT: ${KMS_HTTP_PORT}" -else - print "Using KMS_HTTP_PORT: ${KMS_HTTP_PORT}" -fi - -if [ "${KMS_ADMIN_PORT}" = "" ]; then - export KMS_ADMIN_PORT=`expr $KMS_HTTP_PORT + 1` - print "Setting KMS_ADMIN_PORT: ${KMS_ADMIN_PORT}" -else - print "Using KMS_ADMIN_PORT: ${KMS_ADMIN_PORT}" -fi - -if [ "${KMS_MAX_THREADS}" = "" ]; then - export KMS_MAX_THREADS=1000 - print "Setting KMS_MAX_THREADS: ${KMS_MAX_THREADS}" -else - print "Using KMS_MAX_THREADS: ${KMS_MAX_THREADS}" -fi - -if [ "${KMS_SSL_KEYSTORE_FILE}" = "" ]; then - export KMS_SSL_KEYSTORE_FILE=${HOME}/.keystore - print "Setting KMS_SSL_KEYSTORE_FILE: ${KMS_SSL_KEYSTORE_FILE}" -else - print "Using KMS_SSL_KEYSTORE_FILE: ${KMS_SSL_KEYSTORE_FILE}" -fi - -# If KMS_SSL_KEYSTORE_PASS is explicitly set to "" -# then reset to "password". DO NOT set to "password" if -# variable is NOT defined. -if [ "${KMS_SSL_KEYSTORE_PASS}" = "" ]; then - if [ -n "${KMS_SSL_KEYSTORE_PASS+1}" ]; then - export KMS_SSL_KEYSTORE_PASS=password - print "Setting KMS_SSL_KEYSTORE_PASS: ********" - fi -else - KMS_SSL_KEYSTORE_PASS_DISP=`echo ${KMS_SSL_KEYSTORE_PASS} | sed 's/./*/g'` - print "Using KMS_SSL_KEYSTORE_PASS: ${KMS_SSL_KEYSTORE_PASS_DISP}" -fi - -if [ "${CATALINA_BASE}" = "" ]; then - export CATALINA_BASE=${KMS_HOME}/share/hadoop/kms/tomcat - print "Setting CATALINA_BASE: ${CATALINA_BASE}" -else - print "Using CATALINA_BASE: ${CATALINA_BASE}" -fi - -if [ "${KMS_CATALINA_HOME}" = "" ]; then - export KMS_CATALINA_HOME=${CATALINA_BASE} - print "Setting KMS_CATALINA_HOME: ${KMS_CATALINA_HOME}" -else - print "Using KMS_CATALINA_HOME: ${KMS_CATALINA_HOME}" -fi - -if [ "${CATALINA_OUT}" = "" ]; then - export CATALINA_OUT=${KMS_LOG}/kms-catalina.out - print "Setting CATALINA_OUT: ${CATALINA_OUT}" -else - print "Using CATALINA_OUT: ${CATALINA_OUT}" -fi - -if [ "${CATALINA_PID}" = "" ]; then - export CATALINA_PID=/tmp/kms.pid - print "Setting CATALINA_PID: ${CATALINA_PID}" -else - print "Using CATALINA_PID: ${CATALINA_PID}" -fi - -print diff --git a/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh b/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh index f6ef6a5d0c..e4d4f93f9d 100644 --- a/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh +++ b/hadoop-common-project/hadoop-kms/src/main/sbin/kms.sh @@ -13,32 +13,39 @@ # limitations under the License. # -# resolve links - $0 may be a softlink -PRG="${0}" +function hadoop_usage() +{ + echo "Usage: kms.sh [--config confdir] [--debug] --daemon start|status|stop" + echo " kms.sh [--config confdir] [--debug] COMMAND" + echo " where COMMAND is one of:" + echo " run Start kms in the current window" + echo " run -security Start in the current window with security manager" + echo " start Start kms in a separate window" + echo " start -security Start in a separate window with security manager" + echo " status Return the LSB compliant status" + echo " stop Stop kms, waiting up to 5 seconds for the process to end" + echo " stop n Stop kms, waiting up to n seconds for the process to end" + echo " stop -force Stop kms, wait up to 5 seconds and then use kill -KILL if still running" + echo " stop n -force Stop kms, wait up to n seconds and then use kill -KILL if still running" +} -while [ -h "${PRG}" ]; do - ls=`ls -ld "${PRG}"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "${PRG}"`/"$link" - fi -done - -BASEDIR=`dirname ${PRG}` -BASEDIR=`cd ${BASEDIR}/..;pwd` - -KMS_SILENT=${KMS_SILENT:-true} - -HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-${BASEDIR}/libexec}" -source ${HADOOP_LIBEXEC_DIR}/kms-config.sh - - -if [ "x$JAVA_LIBRARY_PATH" = "x" ]; then - JAVA_LIBRARY_PATH="${HADOOP_LIBEXEC_DIR}/../lib/native/" +# let's locate libexec... +if [[ -n "${HADOOP_PREFIX}" ]]; then + DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" else - JAVA_LIBRARY_PATH="${HADOOP_LIBEXEC_DIR}/../lib/native/:${JAVA_LIBRARY_PATH}" + this="${BASH_SOURCE-$0}" + bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) + DEFAULT_LIBEXEC_DIR="${bin}/../libexec" +fi + +HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}" +# shellcheck disable=SC2034 +HADOOP_NEW_CONFIG=true +if [[ -f "${HADOOP_LIBEXEC_DIR}/kms-config.sh" ]]; then + . "${HADOOP_LIBEXEC_DIR}/kms-config.sh" +else + echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/kms-config.sh." 2>&1 + exit 1 fi # The Java System property 'kms.http.port' it is not used by Kms, @@ -46,38 +53,56 @@ fi # # Mask the trustStorePassword -KMS_SSL_TRUSTSTORE_PASS=`echo $CATALINA_OPTS | grep -o 'trustStorePassword=[^ ]*' | awk -F'=' '{print $2}'` -CATALINA_OPTS_DISP=`echo ${CATALINA_OPTS} | sed -e 's/trustStorePassword=[^ ]*/trustStorePassword=***/'` -print "Using CATALINA_OPTS: ${CATALINA_OPTS_DISP}" +# shellcheck disable=SC2086 +CATALINA_OPTS_DISP="$(echo ${CATALINA_OPTS} | sed -e 's/trustStorePassword=[^ ]*/trustStorePassword=***/')" -catalina_opts="-Dkms.home.dir=${KMS_HOME}"; -catalina_opts="${catalina_opts} -Dkms.config.dir=${KMS_CONFIG}"; -catalina_opts="${catalina_opts} -Dkms.log.dir=${KMS_LOG}"; -catalina_opts="${catalina_opts} -Dkms.temp.dir=${KMS_TEMP}"; -catalina_opts="${catalina_opts} -Dkms.admin.port=${KMS_ADMIN_PORT}"; -catalina_opts="${catalina_opts} -Dkms.http.port=${KMS_HTTP_PORT}"; -catalina_opts="${catalina_opts} -Dkms.max.threads=${KMS_MAX_THREADS}"; -catalina_opts="${catalina_opts} -Dkms.ssl.keystore.file=${KMS_SSL_KEYSTORE_FILE}"; -catalina_opts="${catalina_opts} -Djava.library.path=${JAVA_LIBRARY_PATH}"; +hadoop_debug "Using CATALINA_OPTS: ${CATALINA_OPTS_DISP}" -print "Adding to CATALINA_OPTS: ${catalina_opts}" -print "Found KMS_SSL_KEYSTORE_PASS: `echo ${KMS_SSL_KEYSTORE_PASS} | sed 's/./*/g'`" +# We're using hadoop-common, so set up some stuff it might need: +hadoop_finalize -export CATALINA_OPTS="${CATALINA_OPTS} ${catalina_opts}" +hadoop_verify_logdir + +if [[ $# = 0 ]]; then + case "${HADOOP_DAEMON_MODE}" in + status) + hadoop_status_daemon "${CATALINA_PID}" + exit + ;; + start) + set -- "start" + ;; + stop) + set -- "stop" + ;; + esac +fi + +hadoop_finalize_catalina_opts +export CATALINA_OPTS # A bug in catalina.sh script does not use CATALINA_OPTS for stopping the server # -if [ "${1}" = "stop" ]; then +if [[ "${1}" = "stop" ]]; then export JAVA_OPTS=${CATALINA_OPTS} fi # If ssl, the populate the passwords into ssl-server.xml before starting tomcat -if [ ! "${KMS_SSL_KEYSTORE_PASS}" = "" ] || [ ! "${KMS_SSL_TRUSTSTORE_PASS}" = "" ]; then - # Set a KEYSTORE_PASS if not already set - KMS_SSL_KEYSTORE_PASS=${KMS_SSL_KEYSTORE_PASS:-password} - cat ${CATALINA_BASE}/conf/ssl-server.xml.conf \ - | sed 's/_kms_ssl_keystore_pass_/'${KMS_SSL_KEYSTORE_PASS}'/g' \ - | sed 's/_kms_ssl_truststore_pass_/'${KMS_SSL_TRUSTSTORE_PASS}'/g' > ${CATALINA_BASE}/conf/ssl-server.xml -fi +# +# KMS_SSL_KEYSTORE_PASS is a bit odd. +# if undefined, then the if test will not enable ssl on its own +# if "", set it to "password". +# if custom, use provided password +# +if [[ -f "${HADOOP_CATALINA_HOME}/conf/ssl-server.xml.conf" ]]; then + if [[ -n "${KMS_SSL_KEYSTORE_PASS+x}" ]] || [[ -n "${KMS_SSL_TRUSTSTORE_PASS}" ]]; then + export KMS_SSL_KEYSTORE_PASS=${KMS_SSL_KEYSTORE_PASS:-password} + sed -e 's/_kms_ssl_keystore_pass_/'${KMS_SSL_KEYSTORE_PASS}'/g' \ + -e 's/_kms_ssl_truststore_pass_/'${KMS_SSL_TRUSTSTORE_PASS}'/g' \ + "${HADOOP_CATALINA_HOME}/conf/ssl-server.xml.conf" \ + > "${HADOOP_CATALINA_HOME}/conf/ssl-server.xml" + chmod 700 "${HADOOP_CATALINA_HOME}/conf/ssl-server.xml" >/dev/null 2>&1 + fi +fi -exec ${KMS_CATALINA_HOME}/bin/catalina.sh "$@" +exec "${HADOOP_CATALINA_HOME}/bin/catalina.sh" "$@"