diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
new file mode 100644
index 0000000000..e089b836ed
--- /dev/null
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
@@ -0,0 +1,101 @@
+
+ hadoop-mapreduce-dist
+
+ tar.gz
+
+ true
+
+
+
+ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/target/classes/bin
+ bin
+
+ container-executor
+
+ 0755
+
+
+ hadoop-yarn/bin
+ bin
+
+ *
+
+ 0755
+
+
+ bin
+ bin
+
+ *
+
+ 0755
+
+
+ hadoop-yarn/conf
+ conf
+
+ **/*
+
+
+
+ sources
+
+ **/*.jar
+ **/target/**
+
+ **/bin/*
+ **/scripts/*
+
+ **/dt-*/images/**
+
+ **/file:/**
+ **/SecurityAuth.audit*
+
+
+ assembly/**
+ pom.xml
+ build*.xml
+ ivy.xml
+ ivy/**
+ INSTALL
+ LICENSE.txt
+ mr-client/**
+ hadoop-yarn/**
+ src/**
+
+
+
+ sources
+
+ **/bin/*
+ **/scripts/*
+
+ 0755
+
+
+
+
+
+ org.apache.hadoop:hadoop-yarn-server-tests
+
+
+ modules
+ false
+ false
+
+
+
+
+
+ false
+ /lib
+
+
+ org.apache.hadoop:hadoop-common
+ org.apache.hadoop:hadoop-hdfs
+
+
+
+
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index ba60ce6132..dcaba89bf0 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -445,6 +445,9 @@ Release 0.23.0 - Unreleased
HADOOP-7627. Improve MetricsAsserts to give more understandable output
on failure. (todd)
+ HADOOP-7642. create hadoop-dist module where TAR stitching would happen.
+ (Thomas White via tucu)
+
OPTIMIZATIONS
HADOOP-7333. Performance improvement in PureJavaCrc32. (Eric Caspole
diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml
new file mode 100644
index 0000000000..e804258826
--- /dev/null
+++ b/hadoop-dist/pom.xml
@@ -0,0 +1,144 @@
+
+
+
+ 4.0.0
+
+ org.apache.hadoop
+ hadoop-project
+ 0.24.0-SNAPSHOT
+ ../hadoop-project
+
+ org.apache.hadoop
+ hadoop-dist
+ 0.24.0-SNAPSHOT
+ Apache Hadoop Distribution
+ Apache Hadoop Distribution
+ jar
+
+
+
+
+ org.apache.hadoop
+ hadoop-common
+ provided
+
+
+ org.apache.hadoop
+ hadoop-hdfs
+ provided
+
+
+ org.apache.hadoop
+ hadoop-mapreduce-client-app
+ provided
+
+
+ org.apache.hadoop
+ hadoop-yarn-api
+ provided
+
+
+
+
+
+
+ maven-deploy-plugin
+
+ true
+
+
+
+ org.apache.rat
+ apache-rat-plugin
+
+
+ pom.xml
+
+
+
+
+
+
+
+
+ dist
+
+ false
+
+
+
+
+ org.apache.maven.plugins
+ maven-antrun-plugin
+
+
+ dist
+ package
+
+ run
+
+
+
+
+ run() {
+ echo "\$ ${@}"
+ "${@}"
+ if [ $? != 0 ]; then
+ echo
+ echo "Failed!"
+ echo
+ exit $?
+ fi
+ }
+
+ ROOT=`cd ${basedir}/..;pwd`
+ echo
+ echo "Current directory `pwd`"
+ echo
+ run rm -rf hadoop-${project.version}
+ run mkdir hadoop-${project.version}
+ run cd hadoop-${project.version}
+ run tar xzf $ROOT/hadoop-common-project/hadoop-common/target/hadoop-common-${project.version}.tar.gz --strip-components 1
+ run tar xzf $ROOT/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${project.version}.tar.gz --strip-components 1
+ run tar xzf $ROOT/hadoop-mapreduce-project/target/hadoop-mapreduce-${project.version}.tar.gz --strip-components 1
+ COMMON_LIB=share/hadoop/common/lib
+ MODULES=../../../../modules
+ run ln -s $MODULES/hadoop-mapreduce-client-app-${project.version}.jar $COMMON_LIB
+ run ln -s $MODULES/hadoop-yarn-api-${project.version}.jar $COMMON_LIB
+ run ln -s $MODULES/hadoop-mapreduce-client-common-${project.version}.jar $COMMON_LIB
+ run ln -s $MODULES/hadoop-yarn-common-${project.version}.jar $COMMON_LIB
+ run ln -s $MODULES/hadoop-mapreduce-client-core-${project.version}.jar $COMMON_LIB
+ run ln -s $MODULES/hadoop-yarn-server-common-${project.version}.jar $COMMON_LIB
+ run ln -s $MODULES/hadoop-mapreduce-client-jobclient-${project.version}.jar $COMMON_LIB
+ run cd -
+ run tar czf hadoop-${project.version}.tar.gz hadoop-${project.version}
+ echo
+ echo "Hadoop dist tar available at: ${project.build.directory}/hadoop-${project.version}.tar.gz"
+ echo
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/hadoop-mapreduce-project/INSTALL b/hadoop-mapreduce-project/INSTALL
index 16db5b6dbe..e6de8cb92e 100644
--- a/hadoop-mapreduce-project/INSTALL
+++ b/hadoop-mapreduce-project/INSTALL
@@ -2,49 +2,31 @@ To compile Hadoop Mapreduce next following, do the following:
Step 1) Install dependencies for yarn
-See http://svn.apache.org/repos/asf/hadoop/common/trunk/hadoop-mapreduce/hadoop-yarn/README
+See http://svn.apache.org/repos/asf/hadoop/common/trunk/hadoop-mapreduce-porject/hadoop-yarn/README
Make sure protbuf library is in your library path or set: export LD_LIBRARY_PATH=/usr/local/lib
Step 2) Checkout
svn checkout http://svn.apache.org/repos/asf/hadoop/common/trunk
-Step 3) Build common
+Step 3) Build
-Go to common directory - choose your regular common build command
-Example: mvn clean install package -Pbintar -DskipTests
+Go to common directory - choose your regular common build command. For example:
-Step 4) Build HDFS
-
-Go to hdfs directory
-ant veryclean mvn-install -Dresolvers=internal
-
-Step 5) Build yarn and mapreduce
-
-Go to mapreduce directory
export MAVEN_OPTS=-Xmx512m
+mvn clean package -Pdist -Dtar -DskipTests -Pnative
-mvn clean install assembly:assembly -DskipTests
+You can omit -Pnative it you don't want to build native packages.
-Copy in build.properties if appropriate - make sure eclipse.home not set
-ant veryclean tar -Dresolvers=internal
+Step 4) Untar the tarball from hadoop-dist/target/ into a clean and different
+directory, say YARN_HOME.
-You will see a tarball in
-ls target/hadoop-mapreduce-0.24.0-SNAPSHOT-all.tar.gz
-
-Step 6) Untar the tarball in a clean and different directory.
-say YARN_HOME.
-
-Make sure you aren't picking up avro-1.3.2.jar, remove:
- $HADOOP_COMMON_HOME/share/hadoop/common/lib/avro-1.3.2.jar
- $YARN_HOME/lib/avro-1.3.2.jar
-
-Step 7)
-Install hdfs/common and start hdfs
+Step 5)
+Start hdfs
To run Hadoop Mapreduce next applications:
-Step 8) export the following variables to where you have things installed:
+Step 6) export the following variables to where you have things installed:
You probably want to export these in hadoop-env.sh and yarn-env.sh also.
export HADOOP_MAPRED_HOME=
@@ -54,7 +36,7 @@ export YARN_HOME=directory where you untarred yarn
export HADOOP_CONF_DIR=
export YARN_CONF_DIR=$HADOOP_CONF_DIR
-Step 9) Setup config: for running mapreduce applications, which now are in user land, you need to setup nodemanager with the following configuration in your yarn-site.xml before you start the nodemanager.
+Step 7) Setup config: for running mapreduce applications, which now are in user land, you need to setup nodemanager with the following configuration in your yarn-site.xml before you start the nodemanager.
yarn.nodemanager.aux-services
mapreduce.shuffle
@@ -65,31 +47,21 @@ Step 9) Setup config: for running mapreduce applications, which now are in user
org.apache.hadoop.mapred.ShuffleHandler
-Step 10) Modify mapred-site.xml to use yarn framework
+Step 8) Modify mapred-site.xml to use yarn framework
mapreduce.framework.name
yarn
-Step 11) Create the following symlinks in $HADOOP_COMMON_HOME/share/hadoop/common/lib
+Step 9) cd $YARN_HOME
-ln -s $YARN_HOME/modules/hadoop-mapreduce-client-app-0.24.0-SNAPSHOT.jar .
-ln -s $YARN_HOME/modules/hadoop-yarn-api-0.24.0-SNAPSHOT.jar .
-ln -s $YARN_HOME/modules/hadoop-mapreduce-client-common-0.24.0-SNAPSHOT.jar .
-ln -s $YARN_HOME/modules/hadoop-yarn-common-0.24.0-SNAPSHOT.jar .
-ln -s $YARN_HOME/modules/hadoop-mapreduce-client-core-0.24.0-SNAPSHOT.jar .
-ln -s $YARN_HOME/modules/hadoop-yarn-server-common-0.24.0-SNAPSHOT.jar .
-ln -s $YARN_HOME/modules/hadoop-mapreduce-client-jobclient-0.24.0-SNAPSHOT.jar .
+Step 10) bin/yarn-daemon.sh start resourcemanager
-Step 12) cd $YARN_HOME
+Step 11) bin/yarn-daemon.sh start nodemanager
-Step 13) bin/yarn-daemon.sh start resourcemanager
+Step 12) bin/yarn-daemon.sh start historyserver
-Step 14) bin/yarn-daemon.sh start nodemanager
-
-Step 15) bin/yarn-daemon.sh start historyserver
-
-Step 16) You are all set, an example on how to run a mapreduce job is:
+Step 13) You are all set, an example on how to run a mapreduce job is:
cd $HADOOP_MAPRED_HOME
ant examples -Dresolvers=internal
$HADOOP_COMMON_HOME/bin/hadoop jar $HADOOP_MAPRED_HOME/build/hadoop-mapreduce-examples-0.24.0-SNAPSHOT.jar randomwriter -Dmapreduce.job.user.name=$USER -Dmapreduce.clientfactory.class.name=org.apache.hadoop.mapred.YarnClientFactory -Dmapreduce.randomwriter.bytespermap=10000 -Ddfs.blocksize=536870912 -Ddfs.block.size=536870912 -libjars $YARN_HOME/modules/hadoop-mapreduce-client-jobclient-0.24.0-SNAPSHOT.jar output
diff --git a/hadoop-mapreduce-project/assembly/all.xml b/hadoop-mapreduce-project/assembly/all.xml
index 2121aab64a..e69de29bb2 100644
--- a/hadoop-mapreduce-project/assembly/all.xml
+++ b/hadoop-mapreduce-project/assembly/all.xml
@@ -1,101 +0,0 @@
-
- all
-
- tar.gz
-
- true
-
-
-
- hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/c/container-executor
- bin
-
- container-executor
-
- 0755
-
-
- hadoop-yarn/bin
- bin
-
- *
-
- 0755
-
-
- bin
- bin
-
- *
-
- 0755
-
-
- hadoop-yarn/conf
- conf
-
- **/*
-
-
-
- sources
-
- **/*.jar
- **/target/**
-
- **/bin/*
- **/scripts/*
-
- **/dt-*/images/**
-
- **/file:/**
- **/SecurityAuth.audit*
-
-
- assembly/**
- pom.xml
- build*.xml
- ivy.xml
- ivy/**
- INSTALL
- LICENSE.txt
- mr-client/**
- hadoop-yarn/**
- src/**
-
-
-
- sources
-
- **/bin/*
- **/scripts/*
-
- 0755
-
-
-
-
-
- org.apache.hadoop:hadoop-yarn-server-tests
-
-
- modules
- false
- false
-
-
-
-
-
- false
- /lib
-
-
- org.apache.hadoop:hadoop-common
- org.apache.hadoop:hadoop-hdfs
-
-
-
-
diff --git a/hadoop-mapreduce-project/hadoop-yarn/pom.xml b/hadoop-mapreduce-project/hadoop-yarn/pom.xml
index 051134904c..67a29df246 100644
--- a/hadoop-mapreduce-project/hadoop-yarn/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-yarn/pom.xml
@@ -25,6 +25,7 @@
UTF-8
true
600000
+ ${project.version}
0.24.0-SNAPSHOT
0.24.0-SNAPSHOT
0.24.0-SNAPSHOT
@@ -317,13 +318,33 @@
+ org.apache.maven.plugins
maven-assembly-plugin
+
+
+ org.apache.hadoop
+ hadoop-assemblies
+ ${hadoop.assemblies.version}
+
+
gnu
-
- assembly/all.xml
-
+ false
+ false
+ ${project.artifactId}-${project.version}
+
+ hadoop-mapreduce-dist
+
+
+
+ tar
+ package
+
+ single
+
+
+
maven-antrun-plugin
diff --git a/hadoop-mapreduce-project/pom.xml b/hadoop-mapreduce-project/pom.xml
index 2ad5b3dec0..e88c94602c 100644
--- a/hadoop-mapreduce-project/pom.xml
+++ b/hadoop-mapreduce-project/pom.xml
@@ -26,6 +26,7 @@
UTF-8
true
600000
+ ${project.version}
0.24.0-SNAPSHOT
0.24.0-SNAPSHOT
0.24.0-SNAPSHOT
@@ -320,13 +321,33 @@
+ org.apache.maven.plugins
maven-assembly-plugin
+
+
+ org.apache.hadoop
+ hadoop-assemblies
+ ${hadoop.assemblies.version}
+
+
gnu
-
- assembly/all.xml
-
+ false
+ false
+ ${project.artifactId}-${project.version}
+
+ hadoop-mapreduce-dist
+
+
+
+ tar
+ package
+
+ single
+
+
+
com.atlassian.maven.plugins
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index aa9b43f7fd..b57c53c9f0 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -114,6 +114,21 @@
hadoop-auth
${project.version}
+
+ org.apache.hadoop
+ hadoop-hdfs
+ ${project.version}
+
+
+ org.apache.hadoop
+ hadoop-mapreduce-client-app
+ ${project.version}
+
+
+ org.apache.hadoop
+ hadoop-yarn-api
+ ${project.version}
+
com.google.guava
diff --git a/pom.xml b/pom.xml
index 29adbe95be..f32151bbff 100644
--- a/pom.xml
+++ b/pom.xml
@@ -41,6 +41,7 @@
hadoop-hdfs-project
hadoop-mapreduce-project
hadoop-tools
+ hadoop-dist