diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml new file mode 100644 index 0000000000..e089b836ed --- /dev/null +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml @@ -0,0 +1,101 @@ + + hadoop-mapreduce-dist + + tar.gz + + true + + + + hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/target/classes/bin + bin + + container-executor + + 0755 + + + hadoop-yarn/bin + bin + + * + + 0755 + + + bin + bin + + * + + 0755 + + + hadoop-yarn/conf + conf + + **/* + + + + sources + + **/*.jar + **/target/** + + **/bin/* + **/scripts/* + + **/dt-*/images/** + + **/file:/** + **/SecurityAuth.audit* + + + assembly/** + pom.xml + build*.xml + ivy.xml + ivy/** + INSTALL + LICENSE.txt + mr-client/** + hadoop-yarn/** + src/** + + + + sources + + **/bin/* + **/scripts/* + + 0755 + + + + + + org.apache.hadoop:hadoop-yarn-server-tests + + + modules + false + false + + + + + + false + /lib + + + org.apache.hadoop:hadoop-common + org.apache.hadoop:hadoop-hdfs + + + + diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index ba60ce6132..dcaba89bf0 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -445,6 +445,9 @@ Release 0.23.0 - Unreleased HADOOP-7627. Improve MetricsAsserts to give more understandable output on failure. (todd) + HADOOP-7642. create hadoop-dist module where TAR stitching would happen. + (Thomas White via tucu) + OPTIMIZATIONS HADOOP-7333. Performance improvement in PureJavaCrc32. (Eric Caspole diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml new file mode 100644 index 0000000000..e804258826 --- /dev/null +++ b/hadoop-dist/pom.xml @@ -0,0 +1,144 @@ + + + + 4.0.0 + + org.apache.hadoop + hadoop-project + 0.24.0-SNAPSHOT + ../hadoop-project + + org.apache.hadoop + hadoop-dist + 0.24.0-SNAPSHOT + Apache Hadoop Distribution + Apache Hadoop Distribution + jar + + + + + org.apache.hadoop + hadoop-common + provided + + + org.apache.hadoop + hadoop-hdfs + provided + + + org.apache.hadoop + hadoop-mapreduce-client-app + provided + + + org.apache.hadoop + hadoop-yarn-api + provided + + + + + + + maven-deploy-plugin + + true + + + + org.apache.rat + apache-rat-plugin + + + pom.xml + + + + + + + + + dist + + false + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + dist + package + + run + + + + + run() { + echo "\$ ${@}" + "${@}" + if [ $? != 0 ]; then + echo + echo "Failed!" + echo + exit $? + fi + } + + ROOT=`cd ${basedir}/..;pwd` + echo + echo "Current directory `pwd`" + echo + run rm -rf hadoop-${project.version} + run mkdir hadoop-${project.version} + run cd hadoop-${project.version} + run tar xzf $ROOT/hadoop-common-project/hadoop-common/target/hadoop-common-${project.version}.tar.gz --strip-components 1 + run tar xzf $ROOT/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${project.version}.tar.gz --strip-components 1 + run tar xzf $ROOT/hadoop-mapreduce-project/target/hadoop-mapreduce-${project.version}.tar.gz --strip-components 1 + COMMON_LIB=share/hadoop/common/lib + MODULES=../../../../modules + run ln -s $MODULES/hadoop-mapreduce-client-app-${project.version}.jar $COMMON_LIB + run ln -s $MODULES/hadoop-yarn-api-${project.version}.jar $COMMON_LIB + run ln -s $MODULES/hadoop-mapreduce-client-common-${project.version}.jar $COMMON_LIB + run ln -s $MODULES/hadoop-yarn-common-${project.version}.jar $COMMON_LIB + run ln -s $MODULES/hadoop-mapreduce-client-core-${project.version}.jar $COMMON_LIB + run ln -s $MODULES/hadoop-yarn-server-common-${project.version}.jar $COMMON_LIB + run ln -s $MODULES/hadoop-mapreduce-client-jobclient-${project.version}.jar $COMMON_LIB + run cd - + run tar czf hadoop-${project.version}.tar.gz hadoop-${project.version} + echo + echo "Hadoop dist tar available at: ${project.build.directory}/hadoop-${project.version}.tar.gz" + echo + + + + + + + + + + + + + + + diff --git a/hadoop-mapreduce-project/INSTALL b/hadoop-mapreduce-project/INSTALL index 16db5b6dbe..e6de8cb92e 100644 --- a/hadoop-mapreduce-project/INSTALL +++ b/hadoop-mapreduce-project/INSTALL @@ -2,49 +2,31 @@ To compile Hadoop Mapreduce next following, do the following: Step 1) Install dependencies for yarn -See http://svn.apache.org/repos/asf/hadoop/common/trunk/hadoop-mapreduce/hadoop-yarn/README +See http://svn.apache.org/repos/asf/hadoop/common/trunk/hadoop-mapreduce-porject/hadoop-yarn/README Make sure protbuf library is in your library path or set: export LD_LIBRARY_PATH=/usr/local/lib Step 2) Checkout svn checkout http://svn.apache.org/repos/asf/hadoop/common/trunk -Step 3) Build common +Step 3) Build -Go to common directory - choose your regular common build command -Example: mvn clean install package -Pbintar -DskipTests +Go to common directory - choose your regular common build command. For example: -Step 4) Build HDFS - -Go to hdfs directory -ant veryclean mvn-install -Dresolvers=internal - -Step 5) Build yarn and mapreduce - -Go to mapreduce directory export MAVEN_OPTS=-Xmx512m +mvn clean package -Pdist -Dtar -DskipTests -Pnative -mvn clean install assembly:assembly -DskipTests +You can omit -Pnative it you don't want to build native packages. -Copy in build.properties if appropriate - make sure eclipse.home not set -ant veryclean tar -Dresolvers=internal +Step 4) Untar the tarball from hadoop-dist/target/ into a clean and different +directory, say YARN_HOME. -You will see a tarball in -ls target/hadoop-mapreduce-0.24.0-SNAPSHOT-all.tar.gz - -Step 6) Untar the tarball in a clean and different directory. -say YARN_HOME. - -Make sure you aren't picking up avro-1.3.2.jar, remove: - $HADOOP_COMMON_HOME/share/hadoop/common/lib/avro-1.3.2.jar - $YARN_HOME/lib/avro-1.3.2.jar - -Step 7) -Install hdfs/common and start hdfs +Step 5) +Start hdfs To run Hadoop Mapreduce next applications: -Step 8) export the following variables to where you have things installed: +Step 6) export the following variables to where you have things installed: You probably want to export these in hadoop-env.sh and yarn-env.sh also. export HADOOP_MAPRED_HOME= @@ -54,7 +36,7 @@ export YARN_HOME=directory where you untarred yarn export HADOOP_CONF_DIR= export YARN_CONF_DIR=$HADOOP_CONF_DIR -Step 9) Setup config: for running mapreduce applications, which now are in user land, you need to setup nodemanager with the following configuration in your yarn-site.xml before you start the nodemanager. +Step 7) Setup config: for running mapreduce applications, which now are in user land, you need to setup nodemanager with the following configuration in your yarn-site.xml before you start the nodemanager. yarn.nodemanager.aux-services mapreduce.shuffle @@ -65,31 +47,21 @@ Step 9) Setup config: for running mapreduce applications, which now are in user org.apache.hadoop.mapred.ShuffleHandler -Step 10) Modify mapred-site.xml to use yarn framework +Step 8) Modify mapred-site.xml to use yarn framework mapreduce.framework.name yarn -Step 11) Create the following symlinks in $HADOOP_COMMON_HOME/share/hadoop/common/lib +Step 9) cd $YARN_HOME -ln -s $YARN_HOME/modules/hadoop-mapreduce-client-app-0.24.0-SNAPSHOT.jar . -ln -s $YARN_HOME/modules/hadoop-yarn-api-0.24.0-SNAPSHOT.jar . -ln -s $YARN_HOME/modules/hadoop-mapreduce-client-common-0.24.0-SNAPSHOT.jar . -ln -s $YARN_HOME/modules/hadoop-yarn-common-0.24.0-SNAPSHOT.jar . -ln -s $YARN_HOME/modules/hadoop-mapreduce-client-core-0.24.0-SNAPSHOT.jar . -ln -s $YARN_HOME/modules/hadoop-yarn-server-common-0.24.0-SNAPSHOT.jar . -ln -s $YARN_HOME/modules/hadoop-mapreduce-client-jobclient-0.24.0-SNAPSHOT.jar . +Step 10) bin/yarn-daemon.sh start resourcemanager -Step 12) cd $YARN_HOME +Step 11) bin/yarn-daemon.sh start nodemanager -Step 13) bin/yarn-daemon.sh start resourcemanager +Step 12) bin/yarn-daemon.sh start historyserver -Step 14) bin/yarn-daemon.sh start nodemanager - -Step 15) bin/yarn-daemon.sh start historyserver - -Step 16) You are all set, an example on how to run a mapreduce job is: +Step 13) You are all set, an example on how to run a mapreduce job is: cd $HADOOP_MAPRED_HOME ant examples -Dresolvers=internal $HADOOP_COMMON_HOME/bin/hadoop jar $HADOOP_MAPRED_HOME/build/hadoop-mapreduce-examples-0.24.0-SNAPSHOT.jar randomwriter -Dmapreduce.job.user.name=$USER -Dmapreduce.clientfactory.class.name=org.apache.hadoop.mapred.YarnClientFactory -Dmapreduce.randomwriter.bytespermap=10000 -Ddfs.blocksize=536870912 -Ddfs.block.size=536870912 -libjars $YARN_HOME/modules/hadoop-mapreduce-client-jobclient-0.24.0-SNAPSHOT.jar output diff --git a/hadoop-mapreduce-project/assembly/all.xml b/hadoop-mapreduce-project/assembly/all.xml index 2121aab64a..e69de29bb2 100644 --- a/hadoop-mapreduce-project/assembly/all.xml +++ b/hadoop-mapreduce-project/assembly/all.xml @@ -1,101 +0,0 @@ - - all - - tar.gz - - true - - - - hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/c/container-executor - bin - - container-executor - - 0755 - - - hadoop-yarn/bin - bin - - * - - 0755 - - - bin - bin - - * - - 0755 - - - hadoop-yarn/conf - conf - - **/* - - - - sources - - **/*.jar - **/target/** - - **/bin/* - **/scripts/* - - **/dt-*/images/** - - **/file:/** - **/SecurityAuth.audit* - - - assembly/** - pom.xml - build*.xml - ivy.xml - ivy/** - INSTALL - LICENSE.txt - mr-client/** - hadoop-yarn/** - src/** - - - - sources - - **/bin/* - **/scripts/* - - 0755 - - - - - - org.apache.hadoop:hadoop-yarn-server-tests - - - modules - false - false - - - - - - false - /lib - - - org.apache.hadoop:hadoop-common - org.apache.hadoop:hadoop-hdfs - - - - diff --git a/hadoop-mapreduce-project/hadoop-yarn/pom.xml b/hadoop-mapreduce-project/hadoop-yarn/pom.xml index 051134904c..67a29df246 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/pom.xml +++ b/hadoop-mapreduce-project/hadoop-yarn/pom.xml @@ -25,6 +25,7 @@ UTF-8 true 600000 + ${project.version} 0.24.0-SNAPSHOT 0.24.0-SNAPSHOT 0.24.0-SNAPSHOT @@ -317,13 +318,33 @@ + org.apache.maven.plugins maven-assembly-plugin + + + org.apache.hadoop + hadoop-assemblies + ${hadoop.assemblies.version} + + gnu - - assembly/all.xml - + false + false + ${project.artifactId}-${project.version} + + hadoop-mapreduce-dist + + + + tar + package + + single + + + maven-antrun-plugin diff --git a/hadoop-mapreduce-project/pom.xml b/hadoop-mapreduce-project/pom.xml index 2ad5b3dec0..e88c94602c 100644 --- a/hadoop-mapreduce-project/pom.xml +++ b/hadoop-mapreduce-project/pom.xml @@ -26,6 +26,7 @@ UTF-8 true 600000 + ${project.version} 0.24.0-SNAPSHOT 0.24.0-SNAPSHOT 0.24.0-SNAPSHOT @@ -320,13 +321,33 @@ + org.apache.maven.plugins maven-assembly-plugin + + + org.apache.hadoop + hadoop-assemblies + ${hadoop.assemblies.version} + + gnu - - assembly/all.xml - + false + false + ${project.artifactId}-${project.version} + + hadoop-mapreduce-dist + + + + tar + package + + single + + + com.atlassian.maven.plugins diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index aa9b43f7fd..b57c53c9f0 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -114,6 +114,21 @@ hadoop-auth ${project.version} + + org.apache.hadoop + hadoop-hdfs + ${project.version} + + + org.apache.hadoop + hadoop-mapreduce-client-app + ${project.version} + + + org.apache.hadoop + hadoop-yarn-api + ${project.version} + com.google.guava diff --git a/pom.xml b/pom.xml index 29adbe95be..f32151bbff 100644 --- a/pom.xml +++ b/pom.xml @@ -41,6 +41,7 @@ hadoop-hdfs-project hadoop-mapreduce-project hadoop-tools + hadoop-dist