big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-3366. Mapreduce component should use consistent directory structure layout as HDFS/common (Eric Yang via mahadev)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1215065 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Mahadev Konar 2011-12-16 09:09:28 +00:00
parent c4ff892520
commit 6d551b83de
16 changed files with 130 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
View File

@ -22,7 +22,6 @@
<format>dir</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
<!-- TODO: this layout is wrong. We need module specific bin files in module specific dirs -->
<fileSets>
<fileSet>
<directory>hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/target/native/target/usr/local/bin</directory>
@ -33,7 +32,7 @@
<directory>hadoop-yarn/bin</directory>
<outputDirectory>bin</outputDirectory>
<includes>
<include>*</include>
<include>yarn</include>
</includes>
<fileMode>0755</fileMode>
</fileSet>
@ -41,17 +40,81 @@
<directory>bin</directory>
<outputDirectory>bin</outputDirectory>
<includes>
<include>*</include>
<include>mapred</include>
</includes>
<fileMode>0755</fileMode>
</fileSet>
<fileSet>
<directory>bin</directory>
<outputDirectory>libexec</outputDirectory>
<includes>
<include>mapred-config.sh</include>
</includes>
<fileMode>0755</fileMode>
</fileSet>
<fileSet>
<directory>hadoop-yarn/bin</directory>
<outputDirectory>libexec</outputDirectory>
<includes>
<include>yarn-config.sh</include>
</includes>
<fileMode>0755</fileMode>
</fileSet>
<fileSet>
<directory>hadoop-yarn/bin</directory>
<outputDirectory>sbin</outputDirectory>
<includes>
<include>yarn-daemon.sh</include>
<include>yarn-daemons.sh</include>
<include>start-yarn.sh</include>
<include>stop-yarn.sh</include>
</includes>
<fileMode>0755</fileMode>
</fileSet>
<fileSet>
<directory>hadoop-yarn/conf</directory>
<outputDirectory>conf</outputDirectory>
<outputDirectory>etc/hadoop</outputDirectory>
<includes>
<include>**/*</include>
</includes>
</fileSet>
<fileSet>
<directory>${basedir}</directory>
<outputDirectory>/share/doc/hadoop/${hadoop.component}</outputDirectory>
<includes>
<include>*.txt</include>
</includes>
</fileSet>
<fileSet>
<directory>${project.build.directory}/webapps</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/webapps</outputDirectory>
</fileSet>
<fileSet>
<directory>${basedir}/src/main/conf</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/templates</outputDirectory>
<includes>
<include>*-site.xml</include>
</includes>
</fileSet>
<fileSet>
<directory>${basedir}/src/main/packages/templates/conf</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/templates/conf</outputDirectory>
<includes>
<include>*</include>
</includes>
</fileSet>
<fileSet>
<directory>${basedir}/dev-support/jdiff</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/jdiff</outputDirectory>
</fileSet>
<fileSet>
<directory>${project.build.directory}/site/jdiff/xml</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/jdiff</outputDirectory>
</fileSet>
<fileSet>
<directory>${project.build.directory}/site</directory>
<outputDirectory>/share/doc/hadoop/${hadoop.component}</outputDirectory>
</fileSet>
</fileSets>
<moduleSets>
<moduleSet>
@ -59,7 +122,7 @@
<exclude>org.apache.hadoop:hadoop-yarn-server-tests</exclude>
</excludes>
<binaries>
<outputDirectory>modules</outputDirectory>
<outputDirectory>share/hadoop/${hadoop.component}</outputDirectory>
<includeDependencies>false</includeDependencies>
<unpack>false</unpack>
</binaries>
@ -68,7 +131,7 @@
<dependencySets>
<dependencySet>
<useProjectArtifact>false</useProjectArtifact>
<outputDirectory>/lib</outputDirectory>
<outputDirectory>/share/hadoop/${hadoop.component}/lib</outputDirectory>
<!-- Exclude hadoop artifacts. They will be found via HADOOP* env -->
<excludes>
<exclude>org.apache.hadoop:hadoop-common</exclude>

17
hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
View File

@ -231,6 +231,23 @@ fi
CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/share/hadoop/hdfs'/*'
# put yarn in classpath if present
if [ "$YARN_HOME" = "" ]; then
if [ -d "${HADOOP_PREFIX}/share/hadoop/mapreduce" ]; then
YARN_HOME=$HADOOP_PREFIX
fi
fi
if [ -d "$YARN_HOME/share/hadoop/mapreduce/webapps" ]; then
CLASSPATH=${CLASSPATH}:$YARN_HOME/share/hadoop/mapreduce
fi
if [ -d "$YARN_HOME/share/hadoop/mapreduce/lib" ]; then
CLASSPATH=${CLASSPATH}:$YARN_HOME/share/hadoop/mapreduce/lib'/*'
fi
CLASSPATH=${CLASSPATH}:$YARN_HOME/share/hadoop/mapreduce'/*'
# cygwin path translation
if $cygwin; then
HADOOP_HDFS_HOME=`cygpath -w "$HADOOP_HDFS_HOME"`

12
hadoop-dist/pom.xml
View File

@ -76,6 +76,9 @@
<id>dist</id>
<activation>
<activeByDefault>false</activeByDefault>
<property>
<name>tar|rpm|deb</name>
</property>
</activation>
<build>
<plugins>
@ -114,15 +117,6 @@
run cp -r $ROOT/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${project.version}/* .
run cp -r $ROOT/hadoop-hdfs-project/hadoop-hdfs-httpfs/target/hadoop-hdfs-httpfs-${project.version}/* .
run cp -r $ROOT/hadoop-mapreduce-project/target/hadoop-mapreduce-${project.version}/* .
COMMON_LIB=share/hadoop/common/lib
MODULES=../../../../modules
run ln -s $MODULES/hadoop-mapreduce-client-app-${project.version}.jar $COMMON_LIB
run ln -s $MODULES/hadoop-yarn-api-${project.version}.jar $COMMON_LIB
run ln -s $MODULES/hadoop-mapreduce-client-common-${project.version}.jar $COMMON_LIB
run ln -s $MODULES/hadoop-yarn-common-${project.version}.jar $COMMON_LIB
run ln -s $MODULES/hadoop-mapreduce-client-core-${project.version}.jar $COMMON_LIB
run ln -s $MODULES/hadoop-yarn-server-common-${project.version}.jar $COMMON_LIB
run ln -s $MODULES/hadoop-mapreduce-client-jobclient-${project.version}.jar $COMMON_LIB
echo
echo "Hadoop dist layout available at: ${project.build.directory}/hadoop-${project.version}"
echo

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -323,6 +323,9 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3422. Counter display names are not being picked up. (Jonathan
Eagles via sseth)
MAPREDUCE-3366. Mapreduce component should use consistent directory structure
layout as HDFS/common (Eric Yang via mahadev)
Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES

6
hadoop-mapreduce-project/INSTALL
View File

@ -55,11 +55,11 @@ Step 8) Modify mapred-site.xml to use yarn framework
Step 9) cd $YARN_HOME
Step 10) bin/yarn-daemon.sh start resourcemanager
Step 10) sbin/yarn-daemon.sh start resourcemanager
Step 11) bin/yarn-daemon.sh start nodemanager
Step 11) sbin/yarn-daemon.sh start nodemanager
Step 12) bin/yarn-daemon.sh start historyserver
Step 12) sbin/yarn-daemon.sh start historyserver
Step 13) You are all set, an example on how to run a mapreduce job is:
cd $HADOOP_MAPRED_HOME

2
hadoop-mapreduce-project/hadoop-yarn/bin/slaves.sh
View File

@ -38,7 +38,7 @@ fi
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`
DEFAULT_LIBEXEC_DIR="$bin"
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
. $HADOOP_LIBEXEC_DIR/yarn-config.sh

2
hadoop-mapreduce-project/hadoop-yarn/bin/start-all.sh → hadoop-mapreduce-project/hadoop-yarn/bin/start-yarn.sh
View File

@ -23,7 +23,7 @@ echo "starting yarn daemons"
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`
DEFAULT_LIBEXEC_DIR="$bin"
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
. $HADOOP_LIBEXEC_DIR/yarn-config.sh

2
hadoop-mapreduce-project/hadoop-yarn/bin/stop-all.sh → hadoop-mapreduce-project/hadoop-yarn/bin/stop-yarn.sh
View File

@ -23,7 +23,7 @@ echo "stopping yarn daemons"
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`
DEFAULT_LIBEXEC_DIR="$bin"
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
. $HADOOP_LIBEXEC_DIR/yarn-config.sh

37
hadoop-mapreduce-project/hadoop-yarn/bin/yarn
View File

@ -44,7 +44,7 @@
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`
DEFAULT_LIBEXEC_DIR="$bin"
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
. $HADOOP_LIBEXEC_DIR/yarn-config.sh
@ -109,8 +109,7 @@ if [ ! -d "$HADOOP_CONF_DIR" ]; then
exit 1
fi
CLASSPATH="${HADOOP_CONF_DIR}:${YARN_CONF_DIR}"
CLASSPATH=${CLASSPATH}:${YARN_CLASSPATH}
CLASSPATH="${HADOOP_CONF_DIR}:${YARN_CONF_DIR}:${CLASSPATH}"
CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar
# for developers, add Hadoop classes to CLASSPATH
@ -146,38 +145,6 @@ fi
# so that filenames w/ spaces are handled correctly in loops below
IFS=
# add hadoop-common libs to CLASSPATH
if [ ! -d "$HADOOP_COMMON_HOME" ]; then
if [ -d "$HADOOP_PREFIX" ]; then
export HADOOP_COMMON_HOME=$HADOOP_PREFIX
else
echo No HADOOP_COMMON_HOME set.
echo Please specify it either in yarn-env.sh or in the environment.
exit 1
fi
fi
CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/share/hadoop/common'/*'
CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/share/hadoop/common/lib'/*'
# add hadoop-hdfs libs to CLASSPATH
if [ ! -d "$HADOOP_HDFS_HOME" ]; then
if [ -d "$HADOOP_PREFIX" ]; then
export HADOOP_HDFS_HOME=$HADOOP_PREFIX
else
echo No HADOOP_HDFS_HOME set.
echo Please specify it either in yarn-env.sh or in the environment.
exit 1
fi
fi
CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/share/hadoop/hdfs'/*'
CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib'/*'
# add yarn libs to CLASSPATH
CLASSPATH=${CLASSPATH}:$YARN_HOME/modules'/*'
CLASSPATH=${CLASSPATH}:$YARN_HOME/lib'/*'
# default log directory & file
if [ "$YARN_LOG_DIR" = "" ]; then
YARN_LOG_DIR="$YARN_HOME/logs"

39
hadoop-mapreduce-project/hadoop-yarn/bin/yarn-config.sh
View File

@ -15,29 +15,24 @@
# included in all the hadoop scripts with source command
# should not be executable directly
# also should not be passed any arguments, since we need original $*
# resolve links - $0 may be a softlink
this="$0"
while [ -h "$this" ]; do
ls=`ls -ld "$this"`
link=`expr "$ls" : '.*-> \(.*\)$'`
if expr "$link" : '.*/.*' > /dev/null; then
this="$link"
else
this=`dirname "$this"`/"$link"
fi
done
# convert relative path to absolute path
bin=`dirname "$this"`
script=`basename "$this"`
bin=`which "$0"`
bin=`dirname "${bin}"`
bin=`cd "$bin"; pwd`
this="$bin/$script"
# the root of the Hadoop installation
export YARN_HOME=`dirname "$this"`/..
export HADOOP_PREFIX="${HADOOP_PREFIX:-$bin/..}"
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
if [ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]; then
. ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh
elif [ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]; then
. "$HADOOP_COMMON_HOME"/libexec/hadoop-config.sh
elif [ -e "${HADOOP_HOME}/libexec/hadoop-config.sh" ]; then
. "$HADOOP_HOME"/libexec/hadoop-config.sh
else
echo "Hadoop common not found."
exit
fi
# Same glibc bug that discovered in Hadoop.
# Without this you can see very large vmem settings on containers.
@ -56,7 +51,7 @@ then
fi
# Allow alternate conf dir location.
YARN_CONF_DIR="${YARN_CONF_DIR:-$YARN_HOME/conf}"
YARN_CONF_DIR="${HADOOP_CONF_DIR:-$YARN_HOME/conf}"
#check to see it is specified whether to use the slaves or the
# masters file

2
hadoop-mapreduce-project/hadoop-yarn/bin/yarn-daemon.sh
View File

@ -39,7 +39,7 @@ fi
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`
DEFAULT_LIBEXEC_DIR="$bin"
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
. $HADOOP_LIBEXEC_DIR/yarn-config.sh

2
hadoop-mapreduce-project/hadoop-yarn/bin/yarn-daemons.sh
View File

@ -30,7 +30,7 @@ fi
bin=`dirname "${BASH_SOURCE-$0}"`
bin=`cd "$bin"; pwd`
DEFAULT_LIBEXEC_DIR="$bin"
DEFAULT_LIBEXEC_DIR="$bin"/../libexec
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
. $HADOOP_LIBEXEC_DIR/yarn-config.sh

4
hadoop-mapreduce-project/hadoop-yarn/conf/yarn-env.sh
View File

@ -41,10 +41,6 @@ if [ "$YARN_HEAPSIZE" != "" ]; then
#echo $JAVA_HEAP_MAX
fi
# CLASSPATH initially contains $YARN_CONF_DIR
CLASSPATH="${YARN_CONF_DIR}"
CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar
# so that filenames w/ spaces are handled correctly in loops below
IFS=

4
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
View File

@ -95,8 +95,8 @@ public interface ApplicationConstants {
"$HADOOP_COMMON_HOME/share/hadoop/common/lib/*",
"$HADOOP_HDFS_HOME/share/hadoop/hdfs/*",
"$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*",
"$YARN_HOME/modules/*",
"$YARN_HOME/lib/*"
"$YARN_HOME/share/hadoop/mapreduce/*",
"$YARN_HOME/share/hadoop/mapreduce/lib/*"
};
/**

18
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SingleCluster.apt.vm
View File

@ -171,20 +171,6 @@ Add the following configs to your <<<yarn-site.xml>>>
</property>
+---+
* Create Symlinks.
You will have to create the following symlinks:
+---+
$ cd $HADOOP_COMMON_HOME/share/hadoop/common/lib/
$ ln -s $HADOOP_MAPRED_HOME/modules/hadoop-mapreduce-client-app-*-SNAPSHOT.jar .
$ ln -s $HADOOP_MAPRED_HOME/modules/hadoop-mapreduce-client-jobclient-*-SNAPSHOT.jar .
$ ln -s $HADOOP_MAPRED_HOME/modules/hadoop-mapreduce-client-common-*-SNAPSHOT.jar .
$ ln -s $HADOOP_MAPRED_HOME/modules/hadoop-mapreduce-client-shuffle-*-SNAPSHOT.jar .
$ ln -s $HADOOP_MAPRED_HOME/modules/hadoop-mapreduce-client-core-*-SNAPSHOT.jar .
$ ln -s $HADOOP_MAPRED_HOME/modules/hadoop-yarn-common-*-SNAPSHOT.jar .
$ ln -s $HADOOP_MAPRED_HOME/modules/hadoop-yarn-api-*-SNAPSHOT.jar .
+---+
* Running daemons.
Assuming that the environment variables <<$HADOOP_COMMON_HOME>>, <<$HADOOP_HDFS_HOME>>, <<$HADOO_MAPRED_HOME>>,
@ -195,8 +181,8 @@ $ ln -s $HADOOP_MAPRED_HOME/modules/hadoop-yarn-api-*-SNAPSHOT.jar .
+---+
$ cd $HADOOP_MAPRED_HOME
$ bin/yarn-daemon.sh start resourcemanager
$ bin/yarn-daemon.sh start nodemanager
$ sbin/yarn-daemon.sh start resourcemanager
$ sbin/yarn-daemon.sh start nodemanager
+---+
You should be up and running. You can run randomwriter as:

19
hadoop-mapreduce-project/pom.xml
View File

@ -34,6 +34,8 @@
<test.timeout>600000</test.timeout>
<fork.mode>once</fork.mode>
<mr.basedir>${basedir}</mr.basedir>
<hadoop.component>mapreduce</hadoop.component>
<is.hadoop.component>true</is.hadoop.component>
</properties>
<modules>
@ -321,7 +323,10 @@
<profiles>
<profile>
<id>release</id>
<id>dist</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<build>
<plugins>
<plugin>
@ -336,16 +341,6 @@
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>dist</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
@ -367,7 +362,7 @@
</configuration>
<executions>
<execution>
<id>dist</id>
<id>package-mapreduce</id>
<phase>prepare-package</phase>
<goals>
<goal>single</goal>