diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
index 4868590ac4..57f3c66dad 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
@@ -22,7 +22,6 @@
dir
false
-
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/target/native/target/usr/local/bin
@@ -33,7 +32,7 @@
hadoop-yarn/bin
bin
- *
+ yarn
0755
@@ -41,17 +40,81 @@
bin
bin
- *
+ mapred
+
+ 0755
+
+
+ bin
+ libexec
+
+ mapred-config.sh
+
+ 0755
+
+
+ hadoop-yarn/bin
+ libexec
+
+ yarn-config.sh
+
+ 0755
+
+
+ hadoop-yarn/bin
+ sbin
+
+ yarn-daemon.sh
+ yarn-daemons.sh
+ start-yarn.sh
+ stop-yarn.sh
0755
hadoop-yarn/conf
- conf
+ etc/hadoop
**/*
+
+ ${basedir}
+ /share/doc/hadoop/${hadoop.component}
+
+ *.txt
+
+
+
+ ${project.build.directory}/webapps
+ /share/hadoop/${hadoop.component}/webapps
+
+
+ ${basedir}/src/main/conf
+ /share/hadoop/${hadoop.component}/templates
+
+ *-site.xml
+
+
+
+ ${basedir}/src/main/packages/templates/conf
+ /share/hadoop/${hadoop.component}/templates/conf
+
+ *
+
+
+
+ ${basedir}/dev-support/jdiff
+ /share/hadoop/${hadoop.component}/jdiff
+
+
+ ${project.build.directory}/site/jdiff/xml
+ /share/hadoop/${hadoop.component}/jdiff
+
+
+ ${project.build.directory}/site
+ /share/doc/hadoop/${hadoop.component}
+
@@ -59,7 +122,7 @@
org.apache.hadoop:hadoop-yarn-server-tests
- modules
+ share/hadoop/${hadoop.component}
false
false
@@ -68,7 +131,7 @@
false
- /lib
+ /share/hadoop/${hadoop.component}/lib
org.apache.hadoop:hadoop-common
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
index e53ec737f5..71c9481714 100644
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
@@ -231,6 +231,23 @@ fi
CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/share/hadoop/hdfs'/*'
+# put yarn in classpath if present
+if [ "$YARN_HOME" = "" ]; then
+ if [ -d "${HADOOP_PREFIX}/share/hadoop/mapreduce" ]; then
+ YARN_HOME=$HADOOP_PREFIX
+ fi
+fi
+
+if [ -d "$YARN_HOME/share/hadoop/mapreduce/webapps" ]; then
+ CLASSPATH=${CLASSPATH}:$YARN_HOME/share/hadoop/mapreduce
+fi
+
+if [ -d "$YARN_HOME/share/hadoop/mapreduce/lib" ]; then
+ CLASSPATH=${CLASSPATH}:$YARN_HOME/share/hadoop/mapreduce/lib'/*'
+fi
+
+CLASSPATH=${CLASSPATH}:$YARN_HOME/share/hadoop/mapreduce'/*'
+
# cygwin path translation
if $cygwin; then
HADOOP_HDFS_HOME=`cygpath -w "$HADOOP_HDFS_HOME"`
diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml
index ed6b729a93..93fe32be25 100644
--- a/hadoop-dist/pom.xml
+++ b/hadoop-dist/pom.xml
@@ -76,6 +76,9 @@
dist
false
+
+ tar|rpm|deb
+
@@ -114,15 +117,6 @@
run cp -r $ROOT/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${project.version}/* .
run cp -r $ROOT/hadoop-hdfs-project/hadoop-hdfs-httpfs/target/hadoop-hdfs-httpfs-${project.version}/* .
run cp -r $ROOT/hadoop-mapreduce-project/target/hadoop-mapreduce-${project.version}/* .
- COMMON_LIB=share/hadoop/common/lib
- MODULES=../../../../modules
- run ln -s $MODULES/hadoop-mapreduce-client-app-${project.version}.jar $COMMON_LIB
- run ln -s $MODULES/hadoop-yarn-api-${project.version}.jar $COMMON_LIB
- run ln -s $MODULES/hadoop-mapreduce-client-common-${project.version}.jar $COMMON_LIB
- run ln -s $MODULES/hadoop-yarn-common-${project.version}.jar $COMMON_LIB
- run ln -s $MODULES/hadoop-mapreduce-client-core-${project.version}.jar $COMMON_LIB
- run ln -s $MODULES/hadoop-yarn-server-common-${project.version}.jar $COMMON_LIB
- run ln -s $MODULES/hadoop-mapreduce-client-jobclient-${project.version}.jar $COMMON_LIB
echo
echo "Hadoop dist layout available at: ${project.build.directory}/hadoop-${project.version}"
echo
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index a851daf472..95a98ff344 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -323,6 +323,9 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3422. Counter display names are not being picked up. (Jonathan
Eagles via sseth)
+ MAPREDUCE-3366. Mapreduce component should use consistent directory structure
+ layout as HDFS/common (Eric Yang via mahadev)
+
Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES
diff --git a/hadoop-mapreduce-project/INSTALL b/hadoop-mapreduce-project/INSTALL
index e6de8cb92e..e75b2aff2f 100644
--- a/hadoop-mapreduce-project/INSTALL
+++ b/hadoop-mapreduce-project/INSTALL
@@ -55,11 +55,11 @@ Step 8) Modify mapred-site.xml to use yarn framework
Step 9) cd $YARN_HOME
-Step 10) bin/yarn-daemon.sh start resourcemanager
+Step 10) sbin/yarn-daemon.sh start resourcemanager
-Step 11) bin/yarn-daemon.sh start nodemanager
+Step 11) sbin/yarn-daemon.sh start nodemanager
-Step 12) bin/yarn-daemon.sh start historyserver
+Step 12) sbin/yarn-daemon.sh start historyserver
Step 13) You are all set, an example on how to run a mapreduce job is:
cd $HADOOP_MAPRED_HOME
diff --git a/hadoop-mapreduce-project/hadoop-yarn/bin/slaves.sh b/hadoop-mapreduce-project/hadoop-yarn/bin/slaves.sh
index ee83477901..ee254603d6 100644
--- a/hadoop-mapreduce-project/hadoop-yarn/bin/slaves.sh
+++ b/hadoop-mapreduce-project/hadoop-yarn/bin/slaves.sh
@@ -38,7 +38,7 @@ fi
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`
-DEFAULT_LIBEXEC_DIR="$bin"
+DEFAULT_LIBEXEC_DIR="$bin"/../libexec
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
. $HADOOP_LIBEXEC_DIR/yarn-config.sh
diff --git a/hadoop-mapreduce-project/hadoop-yarn/bin/start-all.sh b/hadoop-mapreduce-project/hadoop-yarn/bin/start-yarn.sh
similarity index 97%
rename from hadoop-mapreduce-project/hadoop-yarn/bin/start-all.sh
rename to hadoop-mapreduce-project/hadoop-yarn/bin/start-yarn.sh
index fa4fcf3d0d..ccd63a4478 100644
--- a/hadoop-mapreduce-project/hadoop-yarn/bin/start-all.sh
+++ b/hadoop-mapreduce-project/hadoop-yarn/bin/start-yarn.sh
@@ -23,7 +23,7 @@ echo "starting yarn daemons"
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`
-DEFAULT_LIBEXEC_DIR="$bin"
+DEFAULT_LIBEXEC_DIR="$bin"/../libexec
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
. $HADOOP_LIBEXEC_DIR/yarn-config.sh
diff --git a/hadoop-mapreduce-project/hadoop-yarn/bin/stop-all.sh b/hadoop-mapreduce-project/hadoop-yarn/bin/stop-yarn.sh
similarity index 97%
rename from hadoop-mapreduce-project/hadoop-yarn/bin/stop-all.sh
rename to hadoop-mapreduce-project/hadoop-yarn/bin/stop-yarn.sh
index 546b67f5c9..c10d1ce7d1 100644
--- a/hadoop-mapreduce-project/hadoop-yarn/bin/stop-all.sh
+++ b/hadoop-mapreduce-project/hadoop-yarn/bin/stop-yarn.sh
@@ -23,7 +23,7 @@ echo "stopping yarn daemons"
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`
-DEFAULT_LIBEXEC_DIR="$bin"
+DEFAULT_LIBEXEC_DIR="$bin"/../libexec
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
. $HADOOP_LIBEXEC_DIR/yarn-config.sh
diff --git a/hadoop-mapreduce-project/hadoop-yarn/bin/yarn b/hadoop-mapreduce-project/hadoop-yarn/bin/yarn
index b8e23a97f5..f5c8c1f8e8 100644
--- a/hadoop-mapreduce-project/hadoop-yarn/bin/yarn
+++ b/hadoop-mapreduce-project/hadoop-yarn/bin/yarn
@@ -44,7 +44,7 @@
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`
-DEFAULT_LIBEXEC_DIR="$bin"
+DEFAULT_LIBEXEC_DIR="$bin"/../libexec
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
. $HADOOP_LIBEXEC_DIR/yarn-config.sh
@@ -109,8 +109,7 @@ if [ ! -d "$HADOOP_CONF_DIR" ]; then
exit 1
fi
-CLASSPATH="${HADOOP_CONF_DIR}:${YARN_CONF_DIR}"
-CLASSPATH=${CLASSPATH}:${YARN_CLASSPATH}
+CLASSPATH="${HADOOP_CONF_DIR}:${YARN_CONF_DIR}:${CLASSPATH}"
CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar
# for developers, add Hadoop classes to CLASSPATH
@@ -146,38 +145,6 @@ fi
# so that filenames w/ spaces are handled correctly in loops below
IFS=
-# add hadoop-common libs to CLASSPATH
-if [ ! -d "$HADOOP_COMMON_HOME" ]; then
- if [ -d "$HADOOP_PREFIX" ]; then
- export HADOOP_COMMON_HOME=$HADOOP_PREFIX
- else
- echo No HADOOP_COMMON_HOME set.
- echo Please specify it either in yarn-env.sh or in the environment.
- exit 1
- fi
-fi
-
-CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/share/hadoop/common'/*'
-CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/share/hadoop/common/lib'/*'
-
-# add hadoop-hdfs libs to CLASSPATH
-if [ ! -d "$HADOOP_HDFS_HOME" ]; then
- if [ -d "$HADOOP_PREFIX" ]; then
- export HADOOP_HDFS_HOME=$HADOOP_PREFIX
- else
- echo No HADOOP_HDFS_HOME set.
- echo Please specify it either in yarn-env.sh or in the environment.
- exit 1
- fi
-fi
-CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/share/hadoop/hdfs'/*'
-CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib'/*'
-
-# add yarn libs to CLASSPATH
-
-CLASSPATH=${CLASSPATH}:$YARN_HOME/modules'/*'
-CLASSPATH=${CLASSPATH}:$YARN_HOME/lib'/*'
-
# default log directory & file
if [ "$YARN_LOG_DIR" = "" ]; then
YARN_LOG_DIR="$YARN_HOME/logs"
diff --git a/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-config.sh b/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-config.sh
index 4371484b86..2757044273 100644
--- a/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-config.sh
+++ b/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-config.sh
@@ -15,29 +15,24 @@
# included in all the hadoop scripts with source command
# should not be executable directly
-# also should not be passed any arguments, since we need original $*
-
-# resolve links - $0 may be a softlink
-
-this="$0"
-while [ -h "$this" ]; do
- ls=`ls -ld "$this"`
- link=`expr "$ls" : '.*-> \(.*\)$'`
- if expr "$link" : '.*/.*' > /dev/null; then
- this="$link"
- else
- this=`dirname "$this"`/"$link"
- fi
-done
-
-# convert relative path to absolute path
-bin=`dirname "$this"`
-script=`basename "$this"`
+bin=`which "$0"`
+bin=`dirname "${bin}"`
bin=`cd "$bin"; pwd`
-this="$bin/$script"
-# the root of the Hadoop installation
-export YARN_HOME=`dirname "$this"`/..
+export HADOOP_PREFIX="${HADOOP_PREFIX:-$bin/..}"
+
+DEFAULT_LIBEXEC_DIR="$bin"/../libexec
+HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
+if [ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]; then
+ . ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh
+elif [ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]; then
+ . "$HADOOP_COMMON_HOME"/libexec/hadoop-config.sh
+elif [ -e "${HADOOP_HOME}/libexec/hadoop-config.sh" ]; then
+ . "$HADOOP_HOME"/libexec/hadoop-config.sh
+else
+ echo "Hadoop common not found."
+ exit
+fi
# Same glibc bug that discovered in Hadoop.
# Without this you can see very large vmem settings on containers.
@@ -56,7 +51,7 @@ then
fi
# Allow alternate conf dir location.
-YARN_CONF_DIR="${YARN_CONF_DIR:-$YARN_HOME/conf}"
+YARN_CONF_DIR="${HADOOP_CONF_DIR:-$YARN_HOME/conf}"
#check to see it is specified whether to use the slaves or the
# masters file
diff --git a/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-daemon.sh b/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-daemon.sh
index 99fcb0a550..6e41f791c3 100644
--- a/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-daemon.sh
+++ b/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-daemon.sh
@@ -39,7 +39,7 @@ fi
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`
-DEFAULT_LIBEXEC_DIR="$bin"
+DEFAULT_LIBEXEC_DIR="$bin"/../libexec
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
. $HADOOP_LIBEXEC_DIR/yarn-config.sh
diff --git a/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-daemons.sh b/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-daemons.sh
index e34e4ca8b1..aafb42b9b1 100644
--- a/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-daemons.sh
+++ b/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-daemons.sh
@@ -30,7 +30,7 @@ fi
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`
-DEFAULT_LIBEXEC_DIR="$bin"
+DEFAULT_LIBEXEC_DIR="$bin"/../libexec
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
. $HADOOP_LIBEXEC_DIR/yarn-config.sh
diff --git a/hadoop-mapreduce-project/hadoop-yarn/conf/yarn-env.sh b/hadoop-mapreduce-project/hadoop-yarn/conf/yarn-env.sh
index b219eddf1a..cfcb250b8e 100644
--- a/hadoop-mapreduce-project/hadoop-yarn/conf/yarn-env.sh
+++ b/hadoop-mapreduce-project/hadoop-yarn/conf/yarn-env.sh
@@ -41,10 +41,6 @@ if [ "$YARN_HEAPSIZE" != "" ]; then
#echo $JAVA_HEAP_MAX
fi
-# CLASSPATH initially contains $YARN_CONF_DIR
-CLASSPATH="${YARN_CONF_DIR}"
-CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar
-
# so that filenames w/ spaces are handled correctly in loops below
IFS=
diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
index c4d0d78ea5..9439e21cfa 100644
--- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
+++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
@@ -95,8 +95,8 @@ public interface ApplicationConstants {
"$HADOOP_COMMON_HOME/share/hadoop/common/lib/*",
"$HADOOP_HDFS_HOME/share/hadoop/hdfs/*",
"$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*",
- "$YARN_HOME/modules/*",
- "$YARN_HOME/lib/*"
+ "$YARN_HOME/share/hadoop/mapreduce/*",
+ "$YARN_HOME/share/hadoop/mapreduce/lib/*"
};
/**
diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SingleCluster.apt.vm b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SingleCluster.apt.vm
index 3d34351708..f4ea1fe69c 100644
--- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SingleCluster.apt.vm
+++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SingleCluster.apt.vm
@@ -171,20 +171,6 @@ Add the following configs to your <<>>
+---+
-* Create Symlinks.
-
- You will have to create the following symlinks:
-
-+---+
-$ cd $HADOOP_COMMON_HOME/share/hadoop/common/lib/
-$ ln -s $HADOOP_MAPRED_HOME/modules/hadoop-mapreduce-client-app-*-SNAPSHOT.jar .
-$ ln -s $HADOOP_MAPRED_HOME/modules/hadoop-mapreduce-client-jobclient-*-SNAPSHOT.jar .
-$ ln -s $HADOOP_MAPRED_HOME/modules/hadoop-mapreduce-client-common-*-SNAPSHOT.jar .
-$ ln -s $HADOOP_MAPRED_HOME/modules/hadoop-mapreduce-client-shuffle-*-SNAPSHOT.jar .
-$ ln -s $HADOOP_MAPRED_HOME/modules/hadoop-mapreduce-client-core-*-SNAPSHOT.jar .
-$ ln -s $HADOOP_MAPRED_HOME/modules/hadoop-yarn-common-*-SNAPSHOT.jar .
-$ ln -s $HADOOP_MAPRED_HOME/modules/hadoop-yarn-api-*-SNAPSHOT.jar .
-+---+
* Running daemons.
Assuming that the environment variables <<$HADOOP_COMMON_HOME>>, <<$HADOOP_HDFS_HOME>>, <<$HADOO_MAPRED_HOME>>,
@@ -195,8 +181,8 @@ $ ln -s $HADOOP_MAPRED_HOME/modules/hadoop-yarn-api-*-SNAPSHOT.jar .
+---+
$ cd $HADOOP_MAPRED_HOME
-$ bin/yarn-daemon.sh start resourcemanager
-$ bin/yarn-daemon.sh start nodemanager
+$ sbin/yarn-daemon.sh start resourcemanager
+$ sbin/yarn-daemon.sh start nodemanager
+---+
You should be up and running. You can run randomwriter as:
diff --git a/hadoop-mapreduce-project/pom.xml b/hadoop-mapreduce-project/pom.xml
index 74970dd5ee..b9e64473cf 100644
--- a/hadoop-mapreduce-project/pom.xml
+++ b/hadoop-mapreduce-project/pom.xml
@@ -34,6 +34,8 @@
600000
once
${basedir}
+ mapreduce
+ true
@@ -321,7 +323,10 @@
- release
+ dist
+
+ false
+
@@ -336,16 +341,6 @@
-
-
-
-
- dist
-
- false
-
-
-
org.apache.maven.plugins
maven-assembly-plugin
@@ -367,7 +362,7 @@
- dist
+ package-mapreduce
prepare-package
single