big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-11485. Pluggable shell integration (aw)

Browse Source
This commit is contained in:
Allen Wittenauer 2015-02-06 13:09:01 -08:00
parent 17165d3df9
commit 5c79439568
11 changed files with 443 additions and 174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml
View File

@ -48,6 +48,11 @@
</includes>
<fileMode>0755</fileMode>
</fileSet>
<fileSet>
<directory>${basedir}/src/main/shellprofile.d</directory>
<outputDirectory>/libexec/shellprofile.d</outputDirectory>
<fileMode>0755</fileMode>
</fileSet>
<fileSet>
<directory>${basedir}/src/main/bin</directory>
<outputDirectory>/sbin</outputDirectory>

5
hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
View File

@ -41,6 +41,11 @@
</includes>
<fileMode>0755</fileMode>
</fileSet>
<fileSet>
<directory>shellprofile.d</directory>
<outputDirectory>libexec/shellprofile.d</outputDirectory>
<fileMode>0755</fileMode>
</fileSet>
<fileSet>
<directory>bin</directory>
<outputDirectory>sbin</outputDirectory>

5
hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
View File

@ -46,6 +46,11 @@
</includes>
<fileMode>0755</fileMode>
</fileSet>
<fileSet>
<directory>hadoop-yarn/shellprofile.d</directory>
<outputDirectory>libexec/shellprofile.d</outputDirectory>
<fileMode>0755</fileMode>
</fileSet>
<fileSet>
<directory>hadoop-yarn/bin</directory>
<outputDirectory>sbin</outputDirectory>

2
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -22,6 +22,8 @@ Trunk (Unreleased)
HADOOP-9044. add FindClass main class to provide classpath checking
of installations (Steve Loughran via aw)
HADOOP-11485. Pluggable shell integration (aw)
IMPROVEMENTS
HADOOP-8017. Configure hadoop-main pom to get rid of M2E plugin execution

24
hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
View File

@ -1,4 +1,4 @@
#
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@ -76,7 +76,7 @@ fi
#
# Let's go! Base definitions so we can move forward
hadoop_bootstrap_init
hadoop_bootstrap
# let's find our conf.
#
@ -158,8 +158,12 @@ while [[ -z "${_hadoop_common_done}" ]]; do
esac
done
#
# Setup the base-line environment
#
hadoop_find_confdir
hadoop_exec_hadoopenv
hadoop_import_shellprofiles
hadoop_exec_userfuncs
#
@ -183,22 +187,20 @@ if declare -F hadoop_subproject_init >/dev/null ; then
hadoop_subproject_init
fi
hadoop_shellprofiles_init
# get the native libs in there pretty quick
hadoop_add_javalibpath "${HADOOP_PREFIX}/build/native"
hadoop_add_javalibpath "${HADOOP_PREFIX}/${HADOOP_COMMON_LIB_NATIVE_DIR}"
hadoop_shellprofiles_nativelib
# get the basic java class path for these subprojects
# in as quickly as possible since other stuff
# will definitely depend upon it.
#
# at some point, this will get replaced with something pluggable
# so that these functions can sit in their projects rather than
# common
#
for i in common hdfs yarn mapred
do
hadoop_add_to_classpath_$i
done
hadoop_add_common_to_classpath
hadoop_shellprofiles_classpath
#
# backwards compatibility. new stuff should

172
hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
View File

@ -50,7 +50,7 @@ function hadoop_deprecate_envvar
fi
}
function hadoop_bootstrap_init
function hadoop_bootstrap
{
# NOTE: This function is not user replaceable.
@ -155,6 +155,87 @@ function hadoop_exec_hadooprc
fi
}
function hadoop_import_shellprofiles
{
local i
local files1
local files2
if [[ -d "${HADOOP_LIBEXEC_DIR}/shellprofile.d" ]]; then
files1=(${HADOOP_LIBEXEC_DIR}/shellprofile.d/*)
else
hadoop_error "WARNING: ${HADOOP_LIBEXEC_DIR}/shellprofile.d doesn't exist. Functionality may not work."
fi
if [[ -d "${HADOOP_CONF_DIR}/shellprofile.d" ]]; then
files2=(${HADOOP_CONF_DIR}/shellprofile.d/*)
fi
for i in "${files1[@]}" "${files2[@]}"
do
if [[ -n "${i}" ]]; then
hadoop_debug "Profiles: importing ${i}"
. "${i}"
fi
done
}
function hadoop_shellprofiles_init
{
local i
for i in ${HADOOP_SHELL_PROFILES}
do
if declare -F _${i}_hadoop_init >/dev/null ; then
hadoop_debug "Profiles: ${i} init"
# shellcheck disable=SC2086
_${i}_hadoop_init
fi
done
}
function hadoop_shellprofiles_classpath
{
local i
for i in ${HADOOP_SHELL_PROFILES}
do
if declare -F _${i}_hadoop_classpath >/dev/null ; then
hadoop_debug "Profiles: ${i} classpath"
# shellcheck disable=SC2086
_${i}_hadoop_classpath
fi
done
}
function hadoop_shellprofiles_nativelib
{
local i
for i in ${HADOOP_SHELL_PROFILES}
do
if declare -F _${i}_hadoop_nativelib >/dev/null ; then
hadoop_debug "Profiles: ${i} nativelib"
# shellcheck disable=SC2086
_${i}_hadoop_nativelib
fi
done
}
function hadoop_shellprofiles_finalize
{
local i
for i in ${HADOOP_SHELL_PROFILES}
do
if declare -F _${i}_hadoop_finalize >/dev/null ; then
hadoop_debug "Profiles: ${i} finalize"
# shellcheck disable=SC2086
_${i}_hadoop_finalize
fi
done
}
function hadoop_basic_init
{
# Some of these are also set in hadoop-env.sh.
@ -164,9 +245,8 @@ function hadoop_basic_init
# but it is important to note that if you update these
# you also need to update hadoop-env.sh as well!!!
# CLASSPATH initially contains $HADOOP_CONF_DIR
CLASSPATH="${HADOOP_CONF_DIR}"
hadoop_debug "Initial CLASSPATH=${HADOOP_CONF_DIR}"
CLASSPATH=""
hadoop_debug "Initialize CLASSPATH"
if [[ -z "${HADOOP_COMMON_HOME}" ]] &&
[[ -d "${HADOOP_PREFIX}/${HADOOP_COMMON_DIR}" ]]; then
@ -360,6 +440,12 @@ function hadoop_add_param
fi
}
function hadoop_add_profile
{
# shellcheck disable=SC2086
hadoop_add_param HADOOP_SHELL_PROFILES $1 $1
}
function hadoop_add_classpath
{
# two params:
@ -451,9 +537,8 @@ function hadoop_add_ldlibpath
export LD_LIBRARY_PATH
}
function hadoop_add_to_classpath_common
function hadoop_add_common_to_classpath
{
#
# get all of the common jars+config in the path
#
@ -471,76 +556,6 @@ function hadoop_add_to_classpath_common
hadoop_add_classpath "${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR}"'/*'
}
function hadoop_add_to_classpath_hdfs
{
#
# get all of the hdfs jars+config in the path
#
# developers
if [[ -n "${HADOOP_ENABLE_BUILD_PATHS}" ]]; then
hadoop_add_classpath "${HADOOP_HDFS_HOME}/hadoop-hdfs/target/classes"
fi
# put hdfs in classpath if present
if [[ -d "${HADOOP_HDFS_HOME}/${HDFS_DIR}/webapps" ]]; then
hadoop_add_classpath "${HADOOP_HDFS_HOME}/${HDFS_DIR}"
fi
hadoop_add_classpath "${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR}"'/*'
hadoop_add_classpath "${HADOOP_HDFS_HOME}/${HDFS_DIR}"'/*'
}
function hadoop_add_to_classpath_yarn
{
local i
#
# get all of the yarn jars+config in the path
#
# developers
if [[ -n "${HADOOP_ENABLE_BUILD_PATHS}" ]]; then
for i in yarn-api yarn-common yarn-mapreduce yarn-master-worker \
yarn-server/yarn-server-nodemanager \
yarn-server/yarn-server-common \
yarn-server/yarn-server-resourcemanager; do
hadoop_add_classpath "${HADOOP_YARN_HOME}/$i/target/classes"
done
hadoop_add_classpath "${HADOOP_YARN_HOME}/build/test/classes"
hadoop_add_classpath "${HADOOP_YARN_HOME}/build/tools"
fi
if [[ -d "${HADOOP_YARN_HOME}/${YARN_DIR}/webapps" ]]; then
hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_DIR}"
fi
hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}"'/*'
hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_DIR}"'/*'
}
function hadoop_add_to_classpath_mapred
{
#
# get all of the mapreduce jars+config in the path
#
# developers
if [[ -n "${HADOOP_ENABLE_BUILD_PATHS}" ]]; then
hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-shuffle/target/classes"
hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-common/target/classes"
hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-hs/target/classes"
hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-hs-plugins/target/classes"
hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-app/target/classes"
hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-jobclient/target/classes"
hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-core/target/classes"
fi
if [[ -d "${HADOOP_MAPRED_HOME}/${MAPRED_DIR}/webapps" ]]; then
hadoop_add_classpath "${HADOOP_MAPRED_HOME}/${MAPRED_DIR}"
fi
hadoop_add_classpath "${HADOOP_MAPRED_HOME}/${MAPRED_LIB_JARS_DIR}"'/*'
hadoop_add_classpath "${HADOOP_MAPRED_HOME}/${MAPRED_DIR}"'/*'
}
function hadoop_add_to_classpath_userpath
{
# Add the user-specified HADOOP_CLASSPATH to the
@ -744,8 +759,9 @@ function hadoop_finalize_catalina_opts
function hadoop_finalize
{
# user classpath gets added at the last minute. this allows
# override of CONF dirs and more
hadoop_shellprofiles_finalize
hadoop_finalize_classpath
hadoop_finalize_libpaths
hadoop_finalize_hadoop_heap

106
hadoop-common-project/hadoop-common/src/main/conf/shellprofile.d/example Normal file
View File

@ -0,0 +1,106 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This is an example shell profile. It does not do anything other than
# show an example of what the general structure and API of the pluggable
# shell profile code looks like.
#
#
#
# First, register the profile:
#
# hadoop_add_profile example
#
#
# This profile name determines what the name of the functions will
# be. The general pattern is _(profilename)_hadoop_(capability). There
# are currently four capabilities:
# * init
# * classpath
# * nativelib
# * finalize
#
# None of these functions are required. Examples of all four follow...
#
# The _hadoop_init function is called near the very beginning of the
# execution cycle. System and site-level shell env vars have been set,
# command line processing finished, etc. Note that the user's .hadooprc
# has not yet been processed. This is to allow them to override anything
# that may be set here or potentially a dependency!
#
# function _example_hadoop_init
# {
# # This example expects a home. So set a default if not set.
# EXAMPLE_HOME="${EXAMPLE_HOME:-/usr/example}"
# }
#
#
# The _hadoop_classpath function is called when the shell code is
# establishing the classpath. This function should use the
# shell hadoop_add_classpath function rather than directly
# manipulating the CLASSPATH variable. This ensures that the
# CLASSPATH does not have duplicates and provides basic
# sanity checks
#
# function _example_hadoop_classpath
# {
# # jars that should be near the front
# hadoop_add_classpath "${EXAMPLE_HOME}/share/pre-jars/*" before
#
# # jars that should be near the back
# hadoop_add_classpath "${EXAMPLE_HOME}/share/post-jars/*" after
# }
#
# The _hadoop_nativelib function is called when the shell code is
# buliding the locations for linkable shared libraries. Depending
# upon needs, there are shell function calls that are useful
# to use here:
#
# hadoop_add_javalibpath will push the path onto the command line
# and into the java.library.path system property. In the majority
# of cases, this should be sufficient, especially if the shared
# library has been linked correctly with $ORIGIN.
#
# hadoop_add_ldlibpath will push the path into the LD_LIBRARY_PATH
# env var. This should be unnecessary for most code.
#
# function _example_hadoop_nativelib
# {
# # our library is standalone, so just need the basic path
# # added. Using after so we are later in the link list
# hadoop_add_javalibpath "${EXAMPLE_HOME}/lib" after
# }
#
# The _hadoop_finalize function is called to finish up whatever
# extra work needs to be done prior to exec'ing java or some other
# binary. This is where command line properties should get added
# and any last minute work. This is called prior to Hadoop common
# which means that one can override any parameters that Hadoop
# would normally put here... so be careful!
#
# Useful functions here include hadoop_add_param and for
# Windows compabitility, hadoop_translate_cygwin_path.
#
# function _example_hadoop_finalize
# {
# # we need a property for our feature
# hadoop_add_param HADOOP_OPTS Dexample.feature "-Dexample.feature=awesome"
# }

36
hadoop-hdfs-project/hadoop-hdfs/src/main/shellprofile.d/hdfs Normal file
View File

@ -0,0 +1,36 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
hadoop_add_profile hdfs
function _hdfs_hadoop_classpath
{
#
# get all of the hdfs jars+config in the path
#
# developers
if [[ -n "${HADOOP_ENABLE_BUILD_PATHS}" ]]; then
hadoop_add_classpath "${HADOOP_HDFS_HOME}/hadoop-hdfs/target/classes"
fi
# put hdfs in classpath if present
if [[ -d "${HADOOP_HDFS_HOME}/${HDFS_DIR}/webapps" ]]; then
hadoop_add_classpath "${HADOOP_HDFS_HOME}/${HDFS_DIR}"
fi
hadoop_add_classpath "${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR}"'/*'
hadoop_add_classpath "${HADOOP_HDFS_HOME}/${HDFS_DIR}"'/*'
}

41
hadoop-mapreduce-project/shellprofile.d/mapreduce Normal file
View File

@ -0,0 +1,41 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
hadoop_add_profile mapred
function _mapred_hadoop_classpath
{
#
# get all of the mapreduce jars+config in the path
#
# developers
if [[ -n "${HADOOP_ENABLE_BUILD_PATHS}" ]]; then
hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-shuffle/target/classes"
hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-common/target/classes"
hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-hs/target/classes"
hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-hs-plugins/target/classes"
hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-app/target/classes"
hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-jobclient/target/classes"
hadoop_add_classpath "${HADOOP_MAPRED_HOME}/hadoop-mapreduce-client-core/target/classes"
fi
if [[ -d "${HADOOP_MAPRED_HOME}/${MAPRED_DIR}/webapps" ]]; then
hadoop_add_classpath "${HADOOP_MAPRED_HOME}/${MAPRED_DIR}"
fi
hadoop_add_classpath "${HADOOP_MAPRED_HOME}/${MAPRED_LIB_JARS_DIR}"'/*'
hadoop_add_classpath "${HADOOP_MAPRED_HOME}/${MAPRED_DIR}"'/*'
}

11
hadoop-yarn-project/hadoop-yarn/bin/yarn
View File

@ -195,17 +195,6 @@ if [[ "${HADOOP_DAEMON_MODE}" != "default" ]]; then
HADOOP_LOGFILE="hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.log"
fi
# Add YARN custom options to comamnd line in case someone actaully
# used these.
YARN_LOG_DIR=$HADOOP_LOG_DIR
hadoop_translate_cygwin_path YARN_LOG_DIR
hadoop_add_param HADOOP_OPTS yarn.log.dir "-Dyarn.log.dir=${YARN_LOG_DIR}"
hadoop_add_param HADOOP_OPTS yarn.log.file "-Dyarn.log.file=${HADOOP_LOGFILE}"
YARN_HOME_DIR=$HADOOP_YARN_HOME
hadoop_translate_cygwin_path YARN_HOME_DIR
hadoop_add_param HADOOP_OPTS yarn.home.dir "-Dyarn.home.dir=${YARN_HOME_DIR}"
hadoop_add_param HADOOP_OPTS yarn.root.logger "-Dyarn.root.logger=${HADOOP_ROOT_LOGGER:-INFO,console}"
hadoop_finalize
if [[ -n "${supportdaemonization}" ]]; then

62
hadoop-yarn-project/hadoop-yarn/shellprofile.d/yarn Normal file
View File

@ -0,0 +1,62 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
hadoop_add_profile yarn
function _yarn_hadoop_classpath
{
local i
#
# get all of the yarn jars+config in the path
#
# developers
if [[ -n "${HADOOP_ENABLE_BUILD_PATHS}" ]]; then
for i in yarn-api yarn-common yarn-mapreduce yarn-master-worker \
yarn-server/yarn-server-nodemanager \
yarn-server/yarn-server-common \
yarn-server/yarn-server-resourcemanager; do
hadoop_add_classpath "${HADOOP_YARN_HOME}/$i/target/classes"
done
hadoop_add_classpath "${HADOOP_YARN_HOME}/build/test/classes"
hadoop_add_classpath "${HADOOP_YARN_HOME}/build/tools"
fi
if [[ -d "${HADOOP_YARN_HOME}/${YARN_DIR}/webapps" ]]; then
hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_DIR}"
fi
hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}"'/*'
hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_DIR}"'/*'
}
function _yarn_hadoop_finalize
{
# Add YARN custom options to comamnd line in case someone actaully
# used these.
#
# Note that we are replacing ' ' with '\ ' so that when we exec
# stuff it works
#
local yld=$HADOOP_LOG_DIR
hadoop_translate_cygwin_path yld
hadoop_add_param HADOOP_OPTS yarn.log.dir "-Dyarn.log.dir=${yld}"
hadoop_add_param HADOOP_OPTS yarn.log.file "-Dyarn.log.file=${HADOOP_LOGFILE}"
local yhd=$HADOOP_YARN_HOME
hadoop_translate_cygwin_path yhd
hadoop_add_param HADOOP_OPTS yarn.home.dir "-Dyarn.home.dir=${yhd}"
hadoop_add_param HADOOP_OPTS yarn.root.logger "-Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
}