big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-7563. Setup HADOOP_HDFS_HOME, HADOOP_MAPRED_HOME and classpath correction. Contributed by Eric Yang.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1161329 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Arun Murthy 2011-08-25 00:06:53 +00:00
parent ebf61db566
commit bb0005cfec
21 changed files with 43 additions and 267 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml
View File

@ -113,7 +113,6 @@
<useProjectArtifact>false</useProjectArtifact>
<excludes>
<exclude>org.apache.ant:*:jar</exclude>
<exclude>org.apache.hadoop:hadoop-*:jar</exclude>
<exclude>jdiff:jdiff:jar</exclude>
</excludes>
</dependencySet>

3
hadoop-common/CHANGES.txt
View File

@ -522,6 +522,9 @@ Trunk (unreleased changes)
HADOOP-7567. 'mvn eclipse:eclipse' fails for hadoop-alfredo (auth).
(Alejandro Abdelnur via tomwhite)
HADOOP-7563. Setup HADOOP_HDFS_HOME, HADOOP_MAPRED_HOME and classpath
correction. (Eric Yang via acmurthy)
Release 0.22.0 - Unreleased
INCOMPATIBLE CHANGES

6
hadoop-common/src/main/bin/hadoop
View File

@ -21,11 +21,7 @@ bin=`which $0`
bin=`dirname ${bin}`
bin=`cd "$bin"; pwd`
if [ -e "$bin/../libexec/hadoop-config.sh" ]; then
. "$bin"/../libexec/hadoop-config.sh
else
. "$bin"/hadoop-config.sh
fi
. "$bin"/../libexec/hadoop-config.sh
function print_usage(){
echo "Usage: hadoop [--config confdir] COMMAND"

128
hadoop-common/src/main/bin/hadoop-config.sh
View File

@ -139,64 +139,19 @@ fi
# CLASSPATH initially contains $HADOOP_CONF_DIR
CLASSPATH="${HADOOP_CONF_DIR}"
# for developers, add Hadoop classes to CLASSPATH
if [ -d "$HADOOP_PREFIX/build/classes" ]; then
CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX/build/classes
fi
if [ -d "$HADOOP_PREFIX/build/webapps" ]; then
CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX/build
fi
if [ -d "$HADOOP_PREFIX/build/test/classes" ]; then
CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX/build/test/classes
fi
if [ -d "$HADOOP_PREFIX/build/test/core/classes" ]; then
CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX/build/test/core/classes
fi
# so that filenames w/ spaces are handled correctly in loops below
IFS=
# for releases, add core hadoop jar & webapps to CLASSPATH
if [ -d "$HADOOP_PREFIX/webapps" ]; then
CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX
if [ -d "$HADOOP_PREFIX/share/hadoop/common/webapps" ]; then
CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX/share/hadoop/common/webapps
fi
if [ -d "$HADOOP_PREFIX/share/hadoop/common/lib" ]; then
for f in $HADOOP_PREFIX/share/hadoop/common/lib/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX/share/hadoop/common/lib'/*'
fi
for f in $HADOOP_PREFIX/share/hadoop/common/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
# for developers, add libs to CLASSPATH
for f in $HADOOP_PREFIX/lib/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
if [ -d "$HADOOP_PREFIX/build/ivy/lib/Hadoop-Common/common" ]; then
for f in $HADOOP_PREFIX/build/ivy/lib/Hadoop-Common/common/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
fi
if [ -d "$HADOOP_PREFIX/build/ivy/lib/hadoop-hdfs/hdfs" ]; then
for f in $HADOOP_PREFIX/build/ivy/lib/hadoop-hdfs/hdfs/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
fi
if [ -d "$HADOOP_PREFIX/build/ivy/lib/Hadoop/mapred" ]; then
for f in $HADOOP_PREFIX/build/ivy/lib/Hadoop/mapred/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
fi
for f in $HADOOP_PREFIX/lib/jsp-2.1/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX/share/hadoop/common'/*'
# add user-specified CLASSPATH last
if [ "$HADOOP_CLASSPATH" != "" ]; then
@ -274,37 +229,20 @@ HADOOP_OPTS="$HADOOP_OPTS -Djava.net.preferIPv4Stack=true"
# put hdfs in classpath if present
if [ "$HADOOP_HDFS_HOME" = "" ]; then
if [ -d "${HADOOP_PREFIX}/share/hadoop/hdfs" ]; then
HADOOP_HDFS_HOME=$HADOOP_PREFIX/share/hadoop/hdfs
#echo Found HDFS installed at $HADOOP_HDFS_HOME
HADOOP_HDFS_HOME=$HADOOP_PREFIX
fi
fi
if [ -d "${HADOOP_HDFS_HOME}" ]; then
if [ -d "$HADOOP_HDFS_HOME/webapps" ]; then
CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME
fi
if [ ! -d "${HADOOP_CONF_DIR}" ] && [ -d "${HADOOP_HDFS_HOME}/conf" ]; then
CLASSPATH=${CLASSPATH}:${HADOOP_HDFS_HOME}/conf
fi
for f in $HADOOP_HDFS_HOME/hadoop-hdfs-*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
# add libs to CLASSPATH
if [ -d "${HADOOP_HDFS_HOME}/lib" ]; then
for f in $HADOOP_HDFS_HOME/lib/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
fi
if [ -d "$HADOOP_HDFS_HOME/build/classes" ]; then
CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/build/classes
fi
if [ -d "$HADOOP_HDFS_HOME/share/hadoop/hdfs/webapps" ]; then
CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/share/hadoop/hdfs
fi
if [ -d "$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib" ]; then
CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib'/*'
fi
CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/share/hadoop/hdfs'/*'
# cygwin path translation
if $cygwin; then
HADOOP_HDFS_HOME=`cygpath -w "$HADOOP_HDFS_HOME"`
@ -313,44 +251,16 @@ fi
# set mapred home if mapred is present
if [ "$HADOOP_MAPRED_HOME" = "" ]; then
if [ -d "${HADOOP_PREFIX}/share/hadoop/mapreduce" ]; then
HADOOP_MAPRED_HOME=$HADOOP_PREFIX/share/hadoop/mapreduce
HADOOP_MAPRED_HOME=$HADOOP_PREFIX
fi
fi
if [ -d "${HADOOP_MAPRED_HOME}" ]; then
if [ -d "$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/webapps" ]; then
CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/webapps
fi
if [ -d "$HADOOP_MAPRED_HOME/webapps" ]; then
CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME
fi
if [ ! -d "${HADOOP_CONF_DIR}" ] && [ -d "${HADOOP_MAPRED_HOME}/conf" ]; then
CLASSPATH=${CLASSPATH}:${HADOOP_MAPRED_HOME}/conf
fi
for f in $HADOOP_MAPRED_HOME/hadoop-mapreduce-*.jar; do
CLASSPATH=${CLASSPATH}:$f
done
if [ -d "${HADOOP_MAPRED_HOME}/lib" ]; then
for f in $HADOOP_MAPRED_HOME/lib/*.jar; do
CLASSPATH=${CLASSPATH}:$f
done
fi
if [ -d "$HADOOP_MAPRED_HOME/build/classes" ]; then
CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build/classes
fi
if [ -d "$HADOOP_MAPRED_HOME/build/tools" ]; then
CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build/tools
fi
for f in $HADOOP_MAPRED_HOME/hadoop-mapreduce-tools-*.jar; do
TOOL_PATH=${TOOL_PATH}:$f;
done
for f in $HADOOP_MAPRED_HOME/build/hadoop-mapreduce-tools-*.jar; do
TOOL_PATH=${TOOL_PATH}:$f;
done
if [ -d "$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib" ]; then
CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib'/*'
fi
# cygwin path translation

6
hadoop-common/src/main/bin/hadoop-daemon.sh
View File

@ -39,11 +39,7 @@ fi
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`
if [ -e "$bin/../libexec/hadoop-config.sh" ]; then
. "$bin"/../libexec/hadoop-config.sh
else
. "$bin"/hadoop-config.sh
fi
. "$bin"/../libexec/hadoop-config.sh
# get arguments

6
hadoop-common/src/main/bin/hadoop-daemons.sh
View File

@ -29,10 +29,6 @@ fi
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`
if [ -e "$bin/../libexec/hadoop-config.sh" ]; then
. "$bin"/../libexec/hadoop-config.sh
else
. "$bin"/hadoop-config.sh
fi
. "$bin"/../libexec/hadoop-config.sh
exec "$bin/slaves.sh" --config $HADOOP_CONF_DIR cd "$HADOOP_PREFIX" \; "$bin/hadoop-daemon.sh" --config $HADOOP_CONF_DIR "$@"

34
hadoop-common/src/main/bin/rcc
View File

@ -50,40 +50,6 @@ fi
JAVA=$JAVA_HOME/bin/java
JAVA_HEAP_MAX=-Xmx1000m
# CLASSPATH initially contains $HADOOP_CONF_DIR
CLASSPATH="${HADOOP_CONF_DIR}"
# for developers, add Hadoop classes to CLASSPATH
if [ -d "$HADOOP_PREFIX/build/classes" ]; then
CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX/build/classes
fi
if [ -d "$HADOOP_PREFIX/build/webapps" ]; then
CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX/build
fi
if [ -d "$HADOOP_PREFIX/build/test/classes" ]; then
CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX/build/test/classes
fi
# so that filenames w/ spaces are handled correctly in loops below
IFS=
# for releases, add core hadoop jar & webapps to CLASSPATH
if [ -d "$HADOOP_PREFIX/webapps" ]; then
CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX
fi
for f in $HADOOP_PREFIX/hadoop-*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
# add libs to CLASSPATH
for f in $HADOOP_PREFIX/lib/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
for f in $HADOOP_PREFIX/lib/jetty-ext/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
# restore ordinary behaviour
unset IFS

6
hadoop-common/src/main/bin/slaves.sh
View File

@ -38,11 +38,7 @@ fi
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`
if [ -e "$bin/../libexec/hadoop-config.sh" ]; then
. "$bin"/../libexec/hadoop-config.sh
else
. "$bin"/hadoop-config.sh
fi
. "$bin"/../libexec/hadoop-config.sh
if [ -f "${HADOOP_CONF_DIR}/hadoop-env.sh" ]; then
. "${HADOOP_CONF_DIR}/hadoop-env.sh"

6
hadoop-common/src/main/bin/start-all.sh
View File

@ -23,11 +23,7 @@ echo "This script is Deprecated. Instead use start-dfs.sh and start-mapred.sh"
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`
if [ -e "$bin/../libexec/hadoop-config.sh" ]; then
. "$bin"/../libexec/hadoop-config.sh
else
. "$bin"/hadoop-config.sh
fi
. "$bin"/../libexec/hadoop-config.sh
# start hdfs daemons if hdfs is present
if [ -f "${HADOOP_HDFS_HOME}"/bin/start-dfs.sh ]; then

6
hadoop-common/src/main/bin/stop-all.sh
View File

@ -23,11 +23,7 @@ echo "This script is Deprecated. Instead use stop-dfs.sh and stop-mapred.sh"
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`
if [ -e "$bin/../libexec/hadoop-config.sh" ]; then
. "$bin"/../libexec/hadoop-config.sh
else
. "$bin"/hadoop-config.sh
fi
. "$bin"/../libexec/hadoop-config.sh
# stop hdfs daemons if hdfs is present
if [ -f "${HADOOP_HDFS_HOME}"/bin/stop-dfs.sh ]; then

6
hadoop-hdfs/src/main/bin/distribute-exclude.sh
View File

@ -36,11 +36,7 @@
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
if [ -e "$bin/../libexec/hdfs-config.sh" ]; then
. "$bin/../libexec/hdfs-config.sh"
else
. "$bin/hdfs-config.sh"
fi
. "$bin/../libexec/hdfs-config.sh"
if [ "$1" = '' ] ; then
"Error: please specify local exclude file as a first argument"

47
hadoop-hdfs/src/main/bin/hdfs
View File

@ -19,11 +19,7 @@ bin=`which $0`
bin=`dirname ${bin}`
bin=`cd "$bin"; pwd`
if [ -e "$bin/../libexec/hdfs-config.sh" ]; then
. "$bin"/../libexec/hdfs-config.sh
else
. "$bin/hdfs-config.sh"
fi
. "$bin"/../libexec/hdfs-config.sh
function print_usage(){
echo "Usage: hdfs [--config confdir] COMMAND"
@ -109,45 +105,6 @@ else
CLASS="$COMMAND"
fi
# for developers, add hdfs classes to CLASSPATH
if [ -d "$HADOOP_HDFS_HOME/build/classes" ]; then
CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/build/classes
fi
if [ -d "$HADOOP_HDFS_HOME/build/web/webapps" ]; then
CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/build/web
fi
if [ -d "$HADOOP_HDFS_HOME/build/test/classes" ]; then
CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/build/test/classes
fi
if [ -d "$HADOOP_HDFS_HOME/build/tools" ]; then
CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/build/tools
fi
if [ -d "$HADOOP_HDFS_HOME/build/ivy/lib/hadoop-hdfs/common" ]; then
for f in $HADOOP_HDFS_HOME/build/ivy/lib/hadoop-hdfs/common/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
fi
if [ -d "$HADOOP_HDFS_HOME/build/ivy/lib/hadoop-hdfs/hdfs" ]; then
for f in $HADOOP_HDFS_HOME/build/ivy/lib/hadoop-hdfs/hdfs/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
fi
# for releases, add core hdfs jar & webapps to CLASSPATH
if [ -d "$HADOOP_PREFIX/share/hadoop/hdfs/webapps" ]; then
CLASSPATH=${CLASSPATH}:$HADOOP_PREFIX/share/hadoop/hdfs
fi
for f in $HADOOP_PREFIX/share/hadoop-hdfs/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
# add libs to CLASSPATH
for f in $HADOOP_PREFIX/lib/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
if $cygwin; then
CLASSPATH=`cygpath -p -w "$CLASSPATH"`
fi
@ -161,7 +118,7 @@ if [ "$starting_secure_dn" = "true" ]; then
HADOOP_SECURE_DN_PID="$HADOOP_PID_DIR/hadoop_secure_dn.pid"
fi
exec "$HADOOP_PREFIX/bin/jsvc" \
exec "$HADOOP_HDFS_HOME/bin/jsvc" \
-Dproc_$COMMAND -outfile "$HADOOP_LOG_DIR/jsvc.out" \
-errfile "$HADOOP_LOG_DIR/jsvc.err" \
-pidfile "$HADOOP_SECURE_DN_PID" \

10
hadoop-hdfs/src/main/bin/hdfs-config.sh
View File

@ -26,12 +26,10 @@ export HADOOP_PREFIX="${HADOOP_PREFIX:-$bin/..}"
if [ -e "$bin/../libexec/hadoop-config.sh" ]; then
. $bin/../libexec/hadoop-config.sh
elif [ -e "${HADOOP_COMMON_HOME}/bin/hadoop-config.sh" ]; then
. "$HADOOP_COMMON_HOME"/bin/hadoop-config.sh
elif [ -e "${HADOOP_HOME}/bin/hadoop-config.sh" ]; then
. "$HADOOP_HOME"/bin/hadoop-config.sh
elif [ -e "${HADOOP_HDFS_HOME}/bin/hadoop-config.sh" ]; then
. "$HADOOP_HDFS_HOME"/bin/hadoop-config.sh
elif [ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]; then
. "$HADOOP_COMMON_HOME"/libexec/hadoop-config.sh
elif [ -e "${HADOOP_HOME}/libexec/hadoop-config.sh" ]; then
. "$HADOOP_HOME"/libexec/hadoop-config.sh
else
echo "Hadoop common not found."
exit

6
hadoop-hdfs/src/main/bin/refresh-namenodes.sh
View File

@ -23,11 +23,7 @@
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
if [ -e "$bin/../libexec/hdfs-config.sh" ]; then
. "$bin/../libexec/hdfs-config.sh"
else
. "$bin/hdfs-config.sh"
fi
. "$bin/../libexec/hdfs-config.sh"
namenodes=$("$HADOOP_PREFIX/bin/hdfs" getconf -nnRpcAddresses)
if [ "$?" != '0' ] ; then errorFlag='1' ;

6
hadoop-hdfs/src/main/bin/start-balancer.sh
View File

@ -18,11 +18,7 @@
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`
if [ -e "$bin/../libexec/hdfs-config.sh" ]; then
. "$bin"/../libexec/hdfs-config.sh
else
. "$bin/hdfs-config.sh"
fi
. "$bin"/../libexec/hdfs-config.sh
# Start balancer daemon.

6
hadoop-hdfs/src/main/bin/start-dfs.sh
View File

@ -25,11 +25,7 @@ usage="Usage: start-dfs.sh [-upgrade|-rollback]"
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`
if [ -e "$bin/../libexec/hdfs-config.sh" ]; then
. "$bin"/../libexec/hdfs-config.sh
else
. "$bin/hdfs-config.sh"
fi
. "$bin"/../libexec/hdfs-config.sh
# get arguments
if [ $# -ge 1 ]; then

7
hadoop-hdfs/src/main/bin/start-secure-dns.sh
View File

@ -22,12 +22,7 @@ usage="Usage (run as root in order to start secure datanodes): start-secure-dns.
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`
if [ -e "$bin/../libexec/hdfs-config.sh" ]; then
. "$bin"/../libexec/hdfs-config.sh
else
. "$bin/hdfs-config.sh"
fi
. "$bin"/../libexec/hdfs-config.sh
if [ "$EUID" -eq 0 ] && [ -n "$HADOOP_SECURE_DN_USER" ]; then
"$HADOOP_PREFIX"/bin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs start datanode $dataStartOpt

7
hadoop-hdfs/src/main/bin/stop-balancer.sh
View File

@ -18,12 +18,7 @@
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`
if [ -e "$bin/../libexec/hdfs-config.sh" ]; then
. "$bin"/../libexec/hdfs-config.sh
else
. "$bin/hdfs-config.sh"
fi
. "$bin"/../libexec/hdfs-config.sh
# Stop balancer daemon.
# Run this on the machine where the balancer is running

6
hadoop-hdfs/src/main/bin/stop-dfs.sh
View File

@ -18,11 +18,7 @@
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`
if [ -e "$bin/../libexec/hdfs-config.sh" ]; then
. "$bin"/../libexec/hdfs-config.sh
else
. "$bin/hdfs-config.sh"
fi
. "$bin"/../libexec/hdfs-config.sh
#---------------------------------------------------------
# namenodes

6
hadoop-hdfs/src/main/bin/stop-secure-dns.sh
View File

@ -22,11 +22,7 @@ usage="Usage (run as root in order to stop secure datanodes): stop-secure-dns.sh
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`
if [ -e "$bin/../libexec/hdfs-config.sh" ]; then
. "$bin"/../libexec/hdfs-config.sh
else
. "$bin/hdfs-config.sh"
fi
. "$bin"/../libexec/hdfs-config.sh
if [ "$EUID" -eq 0 ] && [ -n "$HADOOP_SECURE_DN_USER" ]; then
"$HADOOP_PREFIX"/bin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs stop datanode

1
hadoop-mapreduce/pom.xml
View File

@ -82,6 +82,7 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop-common.version}</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>commons-el</groupId>