From 8a234f394e7a827f22481f618d230c6a630259f3 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Fri, 27 Jan 2012 23:53:35 +0000 Subject: [PATCH] HADOOP-7939. Improve Hadoop subcomponent integration in Hadoop 0.23. (rvs via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1236929 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 + .../src/main/bin/hadoop-config.sh | 81 ++++++++++++++----- .../src/main/sbin/httpfs.sh | 4 +- .../hadoop-hdfs/src/main/bin/hdfs-config.sh | 2 - hadoop-mapreduce-project/bin/mapred | 8 +- hadoop-mapreduce-project/hadoop-yarn/bin/yarn | 4 +- .../hadoop-yarn/bin/yarn-config.sh | 2 - 7 files changed, 69 insertions(+), 34 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 5f1c0340f3..30fa5f25da 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -213,6 +213,8 @@ Release 0.23.1 - Unreleased HADOOP-7919. Remove the unused hadoop.logfile.* properties from the core-default.xml file. (harsh) + HADOOP-7939. Improve Hadoop subcomponent integration in Hadoop 0.23. (rvs via tucu) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh index 71c9481714..c8ecc42664 100644 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh @@ -25,9 +25,21 @@ common_bin=$(cd -P -- "$(dirname -- "$this")" && pwd -P) script="$(basename -- "$this")" this="$common_bin/$script" +[ -f "$common_bin/hadoop-layout.sh" ] && . "$common_bin/hadoop-layout.sh" + +HADOOP_COMMON_DIR=${HADOOP_COMMON_DIR:-"share/hadoop/common"} +HADOOP_COMMON_LIB_JARS_DIR=${HADOOP_COMMON_LIB_JARS_DIR:-"share/hadoop/common/lib"} +HADOOP_COMMON_LIB_NATIVE_DIR=${HADOOP_COMMON_LIB_NATIVE_DIR:-"lib/native"} +HDFS_DIR=${HDFS_DIR:-"share/hadoop/hdfs"} +HDFS_LIB_JARS_DIR=${HDFS_LIB_JARS_DIR:-"share/hadoop/hdfs/lib"} +YARN_DIR=${YARN_DIR:-"share/hadoop/mapreduce"} +YARN_LIB_JARS_DIR=${YARN_LIB_JARS_DIR:-"share/hadoop/mapreduce/lib"} +MAPRED_DIR=${MAPRED_DIR:-"share/hadoop/mapreduce"} +MAPRED_LIB_JARS_DIR=${MAPRED_LIB_JARS_DIR:-"share/hadoop/mapreduce/lib"} + # the root of the Hadoop installation # See HADOOP-6255 for directory structure layout -HADOOP_DEFAULT_PREFIX=`dirname "$this"`/.. +HADOOP_DEFAULT_PREFIX=$(cd -P -- "$common_bin"/.. && pwd -P) HADOOP_PREFIX=${HADOOP_PREFIX:-$HADOOP_DEFAULT_PREFIX} export HADOOP_PREFIX @@ -144,16 +156,22 @@ CLASSPATH="${HADOOP_CONF_DIR}" # so that filenames w/ spaces are handled correctly in loops below IFS= +if [ "$HADOOP_COMMON_HOME" = "" ]; then + if [ -d "${HADOOP_PREFIX}/$HADOOP_COMMON_DIR" ]; then + HADOOP_COMMON_HOME=$HADOOP_PREFIX + fi +fi + # for releases, add core hadoop jar & webapps to CLASSPATH -if [ -d "$HADOOP_PREFIX/share/hadoop/common/webapps" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX/share/hadoop/common/webapps +if [ -d "$HADOOP_COMMON_HOME/$HADOOP_COMMON_DIR/webapps" ]; then + CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/$HADOOP_COMMON_DIR fi -if [ -d "$HADOOP_PREFIX/share/hadoop/common/lib" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX/share/hadoop/common/lib'/*' +if [ -d "$HADOOP_COMMON_HOME/$HADOOP_COMMON_LIB_JARS_DIR" ]; then + CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/$HADOOP_COMMON_LIB_JARS_DIR'/*' fi -CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX/share/hadoop/common'/*' +CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/$HADOOP_COMMON_DIR'/*' # add user-specified CLASSPATH last if [ "$HADOOP_CLASSPATH" != "" ]; then @@ -185,13 +203,13 @@ fi # setup 'java.library.path' for native-hadoop code if necessary -if [ -d "${HADOOP_PREFIX}/build/native" -o -d "${HADOOP_PREFIX}/lib/native" ]; then +if [ -d "${HADOOP_PREFIX}/build/native" -o -d "${HADOOP_PREFIX}/$HADOOP_COMMON_LIB_NATIVE_DIR" ]; then - if [ -d "${HADOOP_PREFIX}/lib/native" ]; then + if [ -d "${HADOOP_PREFIX}/$HADOOP_COMMON_LIB_NATIVE_DIR" ]; then if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then - JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:${HADOOP_PREFIX}/lib/native + JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:${HADOOP_PREFIX}/$HADOOP_COMMON_LIB_NATIVE_DIR else - JAVA_LIBRARY_PATH=${HADOOP_PREFIX}/lib/native + JAVA_LIBRARY_PATH=${HADOOP_PREFIX}/$HADOOP_COMMON_LIB_NATIVE_DIR fi fi fi @@ -216,37 +234,56 @@ HADOOP_OPTS="$HADOOP_OPTS -Djava.net.preferIPv4Stack=true" # put hdfs in classpath if present if [ "$HADOOP_HDFS_HOME" = "" ]; then - if [ -d "${HADOOP_PREFIX}/share/hadoop/hdfs" ]; then + if [ -d "${HADOOP_PREFIX}/$HDFS_DIR" ]; then HADOOP_HDFS_HOME=$HADOOP_PREFIX fi fi -if [ -d "$HADOOP_HDFS_HOME/share/hadoop/hdfs/webapps" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/share/hadoop/hdfs +if [ -d "$HADOOP_HDFS_HOME/$HDFS_DIR/webapps" ]; then + CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/$HDFS_DIR fi -if [ -d "$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib'/*' +if [ -d "$HADOOP_HDFS_HOME/$HDFS_LIB_JARS_DIR" ]; then + CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/$HDFS_LIB_JARS_DIR'/*' fi -CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/share/hadoop/hdfs'/*' +CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/$HDFS_DIR'/*' # put yarn in classpath if present if [ "$YARN_HOME" = "" ]; then - if [ -d "${HADOOP_PREFIX}/share/hadoop/mapreduce" ]; then + if [ -d "${HADOOP_PREFIX}/$YARN_DIR" ]; then YARN_HOME=$HADOOP_PREFIX fi fi -if [ -d "$YARN_HOME/share/hadoop/mapreduce/webapps" ]; then - CLASSPATH=${CLASSPATH}:$YARN_HOME/share/hadoop/mapreduce +if [ -d "$YARN_HOME/$YARN_DIR/webapps" ]; then + CLASSPATH=${CLASSPATH}:$YARN_HOME/$YARN_DIR fi -if [ -d "$YARN_HOME/share/hadoop/mapreduce/lib" ]; then - CLASSPATH=${CLASSPATH}:$YARN_HOME/share/hadoop/mapreduce/lib'/*' +if [ -d "$YARN_HOME/$YARN_LIB_JARS_DIR" ]; then + CLASSPATH=${CLASSPATH}:$YARN_HOME/$YARN_LIB_JARS_DIR'/*' fi -CLASSPATH=${CLASSPATH}:$YARN_HOME/share/hadoop/mapreduce'/*' +CLASSPATH=${CLASSPATH}:$YARN_HOME/$YARN_DIR'/*' + +# put mapred in classpath if present AND different from YARN +if [ "$HADOOP_MAPRED_HOME" = "" ]; then + if [ -d "${HADOOP_PREFIX}/$MAPRED_DIR" ]; then + HADOOP_MAPRED_HOME=$HADOOP_PREFIX + fi +fi + +if [ "$HADOOP_MAPRED_HOME/$MAPRED_DIR" != "$YARN_HOME/$YARN_DIR" ] ; then + if [ -d "$HADOOP_MAPRED_HOME/$MAPRED_DIR/webapps" ]; then + CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/$MAPRED_DIR + fi + + if [ -d "$HADOOP_MAPRED_HOME/$MAPRED_LIB_JARS_DIR" ]; then + CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/$MAPRED_LIB_JARS_DIR'/*' + fi + + CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/$MAPRED_DIR'/*' +fi # cygwin path translation if $cygwin; then diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh index 6566ab25c2..e45bd42d00 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh @@ -55,8 +55,8 @@ if [ "${1}" = "stop" ]; then fi if [ "${HTTPFS_SILENT}" != "true" ]; then - ${BASEDIR}/share/hadoop/httpfs/tomcat/bin/catalina.sh "$@" + ${CATALINA_BASE:-"${BASEDIR}/share/hadoop/httpfs/tomcat"}/bin/catalina.sh "$@" else - ${BASEDIR}/share/hadoop/httpfs/tomcat/bin/catalina.sh "$@" > /dev/null + ${CATALINA_BASE:-"${BASEDIR}/share/hadoop/httpfs/tomcat"}/bin/catalina.sh "$@" > /dev/null fi diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh index 09eec6e5de..2aabf5300b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh @@ -22,8 +22,6 @@ bin=`which "$0"` bin=`dirname "${bin}"` bin=`cd "$bin"; pwd` -export HADOOP_PREFIX="${HADOOP_PREFIX:-$bin/..}" - DEFAULT_LIBEXEC_DIR="$bin"/../libexec HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} if [ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]; then diff --git a/hadoop-mapreduce-project/bin/mapred b/hadoop-mapreduce-project/bin/mapred index 2ffe904917..ff1ebbc67d 100755 --- a/hadoop-mapreduce-project/bin/mapred +++ b/hadoop-mapreduce-project/bin/mapred @@ -91,15 +91,15 @@ if [ -d "$HADOOP_MAPRED_HOME/build/tools" ]; then fi # for releases, add core mapred jar & webapps to CLASSPATH -if [ -d "$HADOOP_PREFIX/share/hadoop/mapreduce/webapps" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX/share/hadoop/mapreduce +if [ -d "$HADOOP_PREFIX/${MAPRED_DIR}/webapps" ]; then + CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX/${MAPRED_DIR} fi -for f in $HADOOP_MAPRED_HOME/share/hadoop-mapreduce/*.jar; do +for f in $HADOOP_MAPRED_HOME/${MAPRED_DIR}/*.jar; do CLASSPATH=${CLASSPATH}:$f; done # add libs to CLASSPATH -for f in $HADOOP_MAPRED_HOME/lib/*.jar; do +for f in $HADOOP_MAPRED_HOME/${MAPRED_LIB_JARS_DIR}/*.jar; do CLASSPATH=${CLASSPATH}:$f; done diff --git a/hadoop-mapreduce-project/hadoop-yarn/bin/yarn b/hadoop-mapreduce-project/hadoop-yarn/bin/yarn index 06f6c29920..7ceac4feae 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/bin/yarn +++ b/hadoop-mapreduce-project/hadoop-yarn/bin/yarn @@ -140,8 +140,8 @@ if [ -d "$YARN_HOME/build/tools" ]; then CLASSPATH=${CLASSPATH}:$YARN_HOME/build/tools fi -CLASSPATH=${CLASSPATH}:$YARN_HOME/share/hadoop/mapreduce/* -CLASSPATH=${CLASSPATH}:$YARN_HOME/share/hadoop/mapreduce/lib/* +CLASSPATH=${CLASSPATH}:$YARN_HOME/${YARN_DIR}/* +CLASSPATH=${CLASSPATH}:$YARN_HOME/${YARN_LIB_JARS_DIR}/* # so that filenames w/ spaces are handled correctly in loops below IFS= diff --git a/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-config.sh b/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-config.sh index 2757044273..934a461b64 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-config.sh +++ b/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-config.sh @@ -19,8 +19,6 @@ bin=`which "$0"` bin=`dirname "${bin}"` bin=`cd "$bin"; pwd` -export HADOOP_PREFIX="${HADOOP_PREFIX:-$bin/..}" - DEFAULT_LIBEXEC_DIR="$bin"/../libexec HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} if [ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]; then