diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 5e57c620c1..194b4be534 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -327,6 +327,9 @@ Trunk (Unreleased) HADOOP-11055. non-daemon pid files are missing (aw) + HADOOP-11022. User replaced functions get lost 2-3 levels deep (e.g., + sbin) (aw) + OPTIMIZATIONS HADOOP-7761. Improve the performance of raw comparisons. (todd) diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh index 40494b3ba4..81a5867e17 100644 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh @@ -156,6 +156,7 @@ done hadoop_find_confdir hadoop_exec_hadoopenv +hadoop_exec_userfuncs # # IMPORTANT! User provided code is now available! diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh index db7dd5e2a9..dfdb101663 100644 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh @@ -104,6 +104,15 @@ function hadoop_exec_hadoopenv fi } +function hadoop_exec_userfuncs +{ + # NOTE: This function is not user replaceable. + + if [[ -e "${HADOOP_CONF_DIR}/hadoop-user-functions.sh" ]]; then + . "${HADOOP_CONF_DIR}/hadoop-user-functions.sh" + fi +} + function hadoop_basic_init { # Some of these are also set in hadoop-env.sh. diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh index eda47c9349..754601a19d 100644 --- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh +++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh @@ -337,84 +337,3 @@ esac # via this special env var: # HADOOP_ENABLE_BUILD_PATHS="true" -# You can do things like replace parts of the shell underbelly. -# Most of this code is in hadoop-functions.sh. -# -# -# For example, if you want to add compression to the rotation -# menthod for the .out files that daemons generate, you can do -# that by redefining the hadoop_rotate_log function by -# uncommenting this code block: - -#function hadoop_rotate_log -#{ -# # -# # log rotation (mainly used for .out files) -# # Users are likely to replace this one for something -# # that gzips or uses dates or who knows what. -# # -# # be aware that &1 and &2 might go through here -# # so don't do anything too crazy... -# # -# local log=$1; -# local num=${2:-5}; -# -# if [[ -f "${log}" ]]; then # rotate logs -# while [[ ${num} -gt 1 ]]; do -# #shellcheck disable=SC2086 -# let prev=${num}-1 -# if [[ -f "${log}.${prev}" ]]; then -# mv "${log}.${prev}" "${log}.${num}" -# fi -# num=${prev} -# done -# mv "${log}" "${log}.${num}" -# gzip -9 "${log}.${num}" -# fi -#} -# -# -# Another example: finding java -# -# By default, Hadoop assumes that $JAVA_HOME is always defined -# outside of its configuration. Eons ago, Apple standardized -# on a helper program called java_home to find it for you. -# -#function hadoop_java_setup -#{ -# -# if [[ -z "${JAVA_HOME}" ]]; then -# case $HADOOP_OS_TYPE in -# Darwin*) -# JAVA_HOME=$(/usr/libexec/java_home) -# ;; -# esac -# fi -# -# # Bail if we did not detect it -# if [[ -z "${JAVA_HOME}" ]]; then -# echo "ERROR: JAVA_HOME is not set and could not be found." 1>&2 -# exit 1 -# fi -# -# if [[ ! -d "${JAVA_HOME}" ]]; then -# echo "ERROR: JAVA_HOME (${JAVA_HOME}) does not exist." 1>&2 -# exit 1 -# fi -# -# JAVA="${JAVA_HOME}/bin/java" -# -# if [[ ! -x ${JAVA} ]]; then -# echo "ERROR: ${JAVA} is not executable." 1>&2 -# exit 1 -# fi -# JAVA_HEAP_MAX=-Xmx1g -# HADOOP_HEAPSIZE=${HADOOP_HEAPSIZE:-128} -# -# # check envvars which might override default args -# if [[ -n "$HADOOP_HEAPSIZE" ]]; then -# JAVA_HEAP_MAX="-Xmx${HADOOP_HEAPSIZE}m" -# fi -#} - - diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-user-functions.sh.example b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-user-functions.sh.example new file mode 100644 index 0000000000..7699c406ba --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-user-functions.sh.example @@ -0,0 +1,94 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +####### +# Advanced Users Only +###### + +# You can do things like replace parts of the shell underbelly. +# Most of this code is in hadoop-functions.sh. +# +# +# For example, if you want to add compression to the rotation +# menthod for the .out files that daemons generate, you can do +# that by redefining the hadoop_rotate_log function by +# uncommenting this code block: + +#function hadoop_rotate_log +#{ +# local log=$1; +# local num=${2:-5}; +# +# if [[ -f "${log}" ]]; then +# while [[ ${num} -gt 1 ]]; do +# #shellcheck disable=SC2086 +# let prev=${num}-1 +# if [[ -f "${log}.${prev}.gz" ]]; then +# mv "${log}.${prev}.gz" "${log}.${num}.gz" +# fi +# num=${prev} +# done +# mv "${log}" "${log}.${num}" +# gzip -9 "${log}.${num}" +# fi +#} +# +# + +# +# Another example: finding java +# +# By default, Hadoop assumes that $JAVA_HOME is always defined +# outside of its configuration. Eons ago, Apple standardized +# on a helper program called java_home to find it for you. +# +#function hadoop_java_setup +#{ +# +# if [[ -z "${JAVA_HOME}" ]]; then +# case $HADOOP_OS_TYPE in +# Darwin*) +# JAVA_HOME=$(/usr/libexec/java_home) +# ;; +# esac +# fi +# +# # Bail if we did not detect it +# if [[ -z "${JAVA_HOME}" ]]; then +# echo "ERROR: JAVA_HOME is not set and could not be found." 1>&2 +# exit 1 +# fi +# +# if [[ ! -d "${JAVA_HOME}" ]]; then +# echo "ERROR: JAVA_HOME (${JAVA_HOME}) does not exist." 1>&2 +# exit 1 +# fi +# +# JAVA="${JAVA_HOME}/bin/java" +# +# if [[ ! -x ${JAVA} ]]; then +# echo "ERROR: ${JAVA} is not executable." 1>&2 +# exit 1 +# fi +# JAVA_HEAP_MAX=-Xmx1g +# HADOOP_HEAPSIZE=${HADOOP_HEAPSIZE:-128} +# +# # check envvars which might override default args +# if [[ -n "$HADOOP_HEAPSIZE" ]]; then +# JAVA_HEAP_MAX="-Xmx${HADOOP_HEAPSIZE}m" +# fi +#} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh index 68240287d9..bb17aa1db6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh @@ -20,8 +20,11 @@ function hadoop_subproject_init { - if [[ -e "${HADOOP_CONF_DIR}/hdfs-env.sh" ]]; then - . "${HADOOP_CONF_DIR}/hdfs-env.sh" + if [[ -z "${HADOOP_HDFS_ENV_PROCESSED}" ]]; then + if [[ -e "${HADOOP_CONF_DIR}/hdfs-env.sh" ]]; then + . "${HADOOP_CONF_DIR}/hdfs-env.sh" + export HADOOP_HDFS_ENV_PROCESSED=true + fi fi # at some point in time, someone thought it would be a good idea to diff --git a/hadoop-mapreduce-project/bin/mapred-config.sh b/hadoop-mapreduce-project/bin/mapred-config.sh index c2681ac66b..cc62b9e59c 100644 --- a/hadoop-mapreduce-project/bin/mapred-config.sh +++ b/hadoop-mapreduce-project/bin/mapred-config.sh @@ -20,8 +20,11 @@ function hadoop_subproject_init { - if [[ -e "${HADOOP_CONF_DIR}/mapred-env.sh" ]]; then - . "${HADOOP_CONF_DIR}/mapred-env.sh" + if [[ -z "${HADOOP_MAPRED_ENV_PROCESSED}" ]]; then + if [[ -e "${HADOOP_CONF_DIR}/mapred-env.sh" ]]; then + . "${HADOOP_CONF_DIR}/mapred-env.sh" + export HADOOP_MAPRED_ENV_PROCESSED=true + fi fi # at some point in time, someone thought it would be a good idea to diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh b/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh index d83e9983d4..372e8efc36 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh @@ -24,10 +24,13 @@ function hadoop_subproject_init # ... # this should get deprecated at some point. - if [[ -e "${YARN_CONF_DIR}/yarn-env.sh" ]]; then - . "${YARN_CONF_DIR}/yarn-env.sh" - elif [[ -e "${HADOOP_CONF_DIR}/yarn-env.sh" ]]; then - . "${HADOOP_CONF_DIR}/yarn-env.sh" + if [[ -z "${HADOOP_YARN_ENV_PROCESSED}" ]]; then + if [[ -e "${YARN_CONF_DIR}/yarn-env.sh" ]]; then + . "${YARN_CONF_DIR}/yarn-env.sh" + elif [[ -e "${HADOOP_CONF_DIR}/yarn-env.sh" ]]; then + . "${HADOOP_CONF_DIR}/yarn-env.sh" + fi + export HADOOP_YARN_ENV_PROCESSED=true fi if [[ -n "${YARN_CONF_DIR}" ]]; then