From 32469f1bdff2bff774674f6a266ec7af215b8cde Mon Sep 17 00:00:00 2001 From: Thomas White Date: Thu, 10 Jun 2010 23:01:33 +0000 Subject: [PATCH] HADOOP-6794. Move configuration and script files post split. Includes HDFS-1181, MAPREDUCE-1033. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@953490 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 2 + bin/hadoop | 8 +- bin/hadoop-config.sh | 85 ++++++++++-------- bin/hdfs | 113 ------------------------ bin/hdfs-config.sh | 33 ------- bin/mapred | 100 --------------------- bin/mapred-config.sh | 33 ------- bin/rcc | 2 +- bin/start-balancer.sh | 25 ------ bin/start-dfs.sh | 52 ----------- bin/start-mapred.sh | 29 ------ bin/stop-balancer.sh | 26 ------ bin/stop-dfs.sh | 28 ------ bin/stop-mapred.sh | 27 ------ conf/capacity-scheduler.xml.template | 126 --------------------------- conf/hadoop-policy.xml.template | 9 ++ conf/hdfs-site.xml.template | 8 -- conf/log4j.properties | 44 +++++++++- conf/mapred-queue-acls.xml.template | 31 ------- conf/mapred-site.xml.template | 8 -- conf/taskcontroller.cfg | 3 - 21 files changed, 111 insertions(+), 681 deletions(-) delete mode 100755 bin/hdfs delete mode 100644 bin/hdfs-config.sh delete mode 100755 bin/mapred delete mode 100644 bin/mapred-config.sh delete mode 100755 bin/start-balancer.sh delete mode 100755 bin/start-dfs.sh delete mode 100755 bin/start-mapred.sh delete mode 100755 bin/stop-balancer.sh delete mode 100755 bin/stop-dfs.sh delete mode 100755 bin/stop-mapred.sh delete mode 100644 conf/capacity-scheduler.xml.template delete mode 100644 conf/hdfs-site.xml.template delete mode 100644 conf/mapred-queue-acls.xml.template delete mode 100644 conf/mapred-site.xml.template delete mode 100644 conf/taskcontroller.cfg diff --git a/CHANGES.txt b/CHANGES.txt index d3c2feb29d..d04fbe5e5c 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -935,6 +935,8 @@ Release 0.21.0 - Unreleased HADOOP-6738. Move cluster_setup.xml from MapReduce to Common. (Tom White via tomwhite) + HADOOP-6794. Move configuration and script files post split. (tomwhite) + OPTIMIZATIONS HADOOP-5595. NameNode does not need to run a replicator to choose a diff --git a/bin/hadoop b/bin/hadoop index 6d34964ea2..10509037cf 100755 --- a/bin/hadoop +++ b/bin/hadoop @@ -53,7 +53,9 @@ case $COMMAND in echo "" #try to locate hdfs and if present, delegate to it. if [ -f "${HADOOP_HDFS_HOME}"/bin/hdfs ]; then - exec "${HADOOP_HDFS_HOME}"/bin/hdfs $* + exec "${HADOOP_HDFS_HOME}"/bin/hdfs $* + elif [ -f "${HADOOP_HOME}"/bin/hdfs ]; then + exec "${HADOOP_HOME}"/bin/hdfs $* else echo "HDFS not found." exit @@ -67,7 +69,9 @@ case $COMMAND in echo "" #try to locate mapred and if present, delegate to it. if [ -f "${HADOOP_MAPRED_HOME}"/bin/mapred ]; then - exec "${HADOOP_MAPRED_HOME}"/bin/mapred $* + exec "${HADOOP_MAPRED_HOME}"/bin/mapred $* + elif [ -f "${HADOOP_HOME}"/bin/mapred ]; then + exec "${HADOOP_HOME}"/bin/mapred $* else echo "MAPRED not found." exit diff --git a/bin/hadoop-config.sh b/bin/hadoop-config.sh index aabeffefcf..e821ed7575 100644 --- a/bin/hadoop-config.sh +++ b/bin/hadoop-config.sh @@ -31,17 +31,17 @@ while [ -h "$this" ]; do done # convert relative path to absolute path -bin=`dirname "$this"` +common_bin=`dirname "$this"` script=`basename "$this"` -bin=`cd "$bin"; pwd` -this="$bin/$script" +common_bin=`cd "$common_bin"; pwd` +this="$common_bin/$script" # the root of the Hadoop installation #TODO: change the env variable when dir structure is changed export HADOOP_HOME=`dirname "$this"`/.. -export HADOOP_CORE_HOME="${HADOOP_HOME}" +export HADOOP_COMMON_HOME="${HADOOP_HOME}" #export HADOOP_HOME=`dirname "$this"`/../.. -#export HADOOP_CORE_HOME="${HADOOP_CORE_HOME:-`dirname "$this"`/..}" +#export HADOOP_COMMON_HOME="${HADOOP_COMMON_HOME:-`dirname "$this"`/..}" #check to see if the conf dir is given as an optional argument if [ $# -gt 1 ] @@ -106,54 +106,54 @@ CLASSPATH="${HADOOP_CONF_DIR}" CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar # for developers, add Hadoop classes to CLASSPATH -if [ -d "$HADOOP_CORE_HOME/build/classes" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_CORE_HOME/build/classes +if [ -d "$HADOOP_COMMON_HOME/build/classes" ]; then + CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/build/classes fi -if [ -d "$HADOOP_CORE_HOME/build/webapps" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_CORE_HOME/build +if [ -d "$HADOOP_COMMON_HOME/build/webapps" ]; then + CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/build fi -if [ -d "$HADOOP_CORE_HOME/build/test/classes" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_CORE_HOME/build/test/classes +if [ -d "$HADOOP_COMMON_HOME/build/test/classes" ]; then + CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/build/test/classes fi -if [ -d "$HADOOP_CORE_HOME/build/test/core/classes" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_CORE_HOME/build/test/core/classes +if [ -d "$HADOOP_COMMON_HOME/build/test/core/classes" ]; then + CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/build/test/core/classes fi # so that filenames w/ spaces are handled correctly in loops below IFS= # for releases, add core hadoop jar & webapps to CLASSPATH -if [ -d "$HADOOP_CORE_HOME/webapps" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_CORE_HOME +if [ -d "$HADOOP_COMMON_HOME/webapps" ]; then + CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME fi -for f in $HADOOP_CORE_HOME/hadoop-core-*.jar; do +for f in $HADOOP_COMMON_HOME/hadoop-*.jar; do CLASSPATH=${CLASSPATH}:$f; done # add libs to CLASSPATH -for f in $HADOOP_CORE_HOME/lib/*.jar; do +for f in $HADOOP_COMMON_HOME/lib/*.jar; do CLASSPATH=${CLASSPATH}:$f; done -if [ -d "$HADOOP_CORE_HOME/build/ivy/lib/Hadoop-Core/common" ]; then -for f in $HADOOP_CORE_HOME/build/ivy/lib/Hadoop-Core/common/*.jar; do +if [ -d "$HADOOP_COMMON_HOME/build/ivy/lib/Hadoop-Common/common" ]; then +for f in $HADOOP_COMMON_HOME/build/ivy/lib/Hadoop-Common/common/*.jar; do CLASSPATH=${CLASSPATH}:$f; done fi -if [ -d "$HADOOP_CORE_HOME/build/ivy/lib/Hadoop-Hdfs/common" ]; then -for f in $HADOOP_CORE_HOME/build/ivy/lib/Hadoop-Hdfs/common/*.jar; do +if [ -d "$HADOOP_COMMON_HOME/build/ivy/lib/Hadoop-Hdfs/common" ]; then +for f in $HADOOP_COMMON_HOME/build/ivy/lib/Hadoop-Hdfs/common/*.jar; do CLASSPATH=${CLASSPATH}:$f; done fi -if [ -d "$HADOOP_CORE_HOME/build/ivy/lib/Hadoop/common" ]; then -for f in $HADOOP_CORE_HOME/build/ivy/lib/Hadoop/common/*.jar; do +if [ -d "$HADOOP_COMMON_HOME/build/ivy/lib/Hadoop/common" ]; then +for f in $HADOOP_COMMON_HOME/build/ivy/lib/Hadoop/common/*.jar; do CLASSPATH=${CLASSPATH}:$f; done fi -for f in $HADOOP_CORE_HOME/lib/jsp-2.1/*.jar; do +for f in $HADOOP_COMMON_HOME/lib/jsp-2.1/*.jar; do CLASSPATH=${CLASSPATH}:$f; done @@ -180,23 +180,23 @@ unset IFS # cygwin path translation if $cygwin; then - HADOOP_CORE_HOME=`cygpath -w "$HADOOP_CORE_HOME"` + HADOOP_COMMON_HOME=`cygpath -w "$HADOOP_COMMON_HOME"` HADOOP_LOG_DIR=`cygpath -w "$HADOOP_LOG_DIR"` fi # setup 'java.library.path' for native-hadoop code if necessary JAVA_LIBRARY_PATH='' -if [ -d "${HADOOP_CORE_HOME}/build/native" -o -d "${HADOOP_CORE_HOME}/lib/native" ]; then +if [ -d "${HADOOP_COMMON_HOME}/build/native" -o -d "${HADOOP_COMMON_HOME}/lib/native" ]; then JAVA_PLATFORM=`CLASSPATH=${CLASSPATH} ${JAVA} -Xmx32m ${HADOOP_JAVA_PLATFORM_OPTS} org.apache.hadoop.util.PlatformName | sed -e "s/ /_/g"` - if [ -d "$HADOOP_CORE_HOME/build/native" ]; then - JAVA_LIBRARY_PATH=${HADOOP_CORE_HOME}/build/native/${JAVA_PLATFORM}/lib + if [ -d "$HADOOP_COMMON_HOME/build/native" ]; then + JAVA_LIBRARY_PATH=${HADOOP_COMMON_HOME}/build/native/${JAVA_PLATFORM}/lib fi - if [ -d "${HADOOP_CORE_HOME}/lib/native" ]; then + if [ -d "${HADOOP_COMMON_HOME}/lib/native" ]; then if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then - JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:${HADOOP_CORE_HOME}/lib/native/${JAVA_PLATFORM} + JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:${HADOOP_COMMON_HOME}/lib/native/${JAVA_PLATFORM} else - JAVA_LIBRARY_PATH=${HADOOP_CORE_HOME}/lib/native/${JAVA_PLATFORM} + JAVA_LIBRARY_PATH=${HADOOP_COMMON_HOME}/lib/native/${JAVA_PLATFORM} fi fi fi @@ -208,7 +208,7 @@ fi HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR" HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.file=$HADOOP_LOGFILE" -HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.home.dir=$HADOOP_CORE_HOME" +HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.home.dir=$HADOOP_COMMON_HOME" HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.id.str=$HADOOP_IDENT_STRING" HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.root.logger=${HADOOP_ROOT_LOGGER:-INFO,console}" if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then @@ -225,6 +225,15 @@ if [ "$HADOOP_HDFS_HOME" = "" ]; then fi if [ -d "${HADOOP_HDFS_HOME}" ]; then + + if [ -d "$HADOOP_HDFS_HOME/webapps" ]; then + CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME + fi + + if [ -d "${HADOOP_HDFS_HOME}/conf" ]; then + CLASSPATH=${CLASSPATH}:${HADOOP_HDFS_HOME}/conf + fi + for f in $HADOOP_HDFS_HOME/hadoop-hdfs-*.jar; do CLASSPATH=${CLASSPATH}:$f; done @@ -253,6 +262,15 @@ if [ "$HADOOP_MAPRED_HOME" = "" ]; then fi if [ -d "${HADOOP_MAPRED_HOME}" ]; then + + if [ -d "$HADOOP_MAPRED_HOME/webapps" ]; then + CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME + fi + + if [ -d "${HADOOP_MAPRED_HOME}/conf" ]; then + CLASSPATH=${CLASSPATH}:${HADOOP_MAPRED_HOME}/conf + fi + for f in $HADOOP_MAPRED_HOME/hadoop-mapred-*.jar; do CLASSPATH=${CLASSPATH}:$f done @@ -284,6 +302,3 @@ if $cygwin; then fi -# TODO:remove this when dir structure is changed -#export HADOOP_HDFS_HOME=$HADOOP_HOME -#export HADOOP_MAPRED_HOME=$HADOOP_HOME diff --git a/bin/hdfs b/bin/hdfs deleted file mode 100755 index 1bc6565cd0..0000000000 --- a/bin/hdfs +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -bin=`dirname "$0"` -bin=`cd "$bin"; pwd` - -. "$bin"/hdfs-config.sh - -function print_usage(){ - echo "Usage: hdfs [--config confdir] COMMAND" - echo " where COMMAND is one of:" - echo " namenode -format format the DFS filesystem" - echo " secondarynamenode run the DFS secondary namenode" - echo " namenode run the DFS namenode" - echo " datanode run a DFS datanode" - echo " dfsadmin run a DFS admin client" - echo " fsck run a DFS filesystem checking utility" - echo " balancer run a cluster balancing utility" - echo " jmxget get JMX exported values from NameNode or DataNode." - echo " oiv apply the offline fsimage viewer to an fsimage" - echo " fetchdt fetch a delegation token from the NameNode" - echo " Use -help to see options" - echo "" - echo "Most commands print help when invoked w/o parameters." -} - -if [ $# = 0 ]; then - print_usage - exit -fi - -COMMAND=$1 -shift - -if [ "$COMMAND" = "namenode" ] ; then - CLASS='org.apache.hadoop.hdfs.server.namenode.NameNode' - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_NAMENODE_OPTS" -elif [ "$COMMAND" = "secondarynamenode" ] ; then - CLASS='org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode' - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_SECONDARYNAMENODE_OPTS" -elif [ "$COMMAND" = "datanode" ] ; then - CLASS='org.apache.hadoop.hdfs.server.datanode.DataNode' - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_DATANODE_OPTS" -elif [ "$COMMAND" = "dfs" ] ; then - CLASS=org.apache.hadoop.fs.FsShell - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" -elif [ "$COMMAND" = "dfsadmin" ] ; then - CLASS=org.apache.hadoop.hdfs.tools.DFSAdmin - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" -elif [ "$COMMAND" = "fsck" ] ; then - CLASS=org.apache.hadoop.hdfs.tools.DFSck - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" -elif [ "$COMMAND" = "balancer" ] ; then - CLASS=org.apache.hadoop.hdfs.server.balancer.Balancer - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_BALANCER_OPTS" -elif [ "$COMMAND" = "jmxget" ] ; then - CLASS=org.apache.hadoop.hdfs.tools.JMXGet -elif [ "$COMMAND" = "oiv" ] ; then - CLASS=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewer -elif [ "$COMMAND" = "fetchdt" ] ; then - CLASS=org.apache.hadoop.hdfs.tools.DelegationTokenFetcher -else - echo $COMMAND - invalid command - print_usage - exit -fi - -# for developers, add hdfs classes to CLASSPATH -if [ -d "$HADOOP_HDFS_HOME/build/classes" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/build/classes -fi -if [ -d "$HADOOP_HDFS_HOME/build/webapps" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/build -fi -if [ -d "$HADOOP_HDFS_HOME/build/test/classes" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/build/test/classes -fi -if [ -d "$HADOOP_HDFS_HOME/build/tools" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/build/tools -fi - -# for releases, add core hdfs jar & webapps to CLASSPATH -if [ -d "$HADOOP_HDFS_HOME/webapps" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME -fi -for f in $HADOOP_HDFS_HOME/hadoop-hdfs-*.jar; do - CLASSPATH=${CLASSPATH}:$f; -done - -# add libs to CLASSPATH -for f in $HADOOP_HDFS_HOME/lib/*.jar; do - CLASSPATH=${CLASSPATH}:$f; -done - -if $cygwin; then - CLASSPATH=`cygpath -p -w "$CLASSPATH"` -fi -export CLASSPATH=$CLASSPATH -exec "$JAVA" $JAVA_HEAP_MAX $HADOOP_OPTS $CLASS "$@" diff --git a/bin/hdfs-config.sh b/bin/hdfs-config.sh deleted file mode 100644 index 9f7ad736e9..0000000000 --- a/bin/hdfs-config.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# included in all the hdfs scripts with source command -# should not be executed directly - -bin=`dirname "$0"` -bin=`cd "$bin"; pwd` - -#TODO: change the env variable when directory structure is changed -export HADOOP_CORE_HOME="${HADOOP_CORE_HOME:-$bin/..}" -#export HADOOP_CORE_HOME="${HADOOP_CORE_HOME:-$bin/../../core}" - -if [ -d "${HADOOP_CORE_HOME}" ]; then - . "$HADOOP_CORE_HOME"/bin/hadoop-config.sh -else - echo "Hadoop core not found." - exit -fi diff --git a/bin/mapred b/bin/mapred deleted file mode 100755 index c56a19da23..0000000000 --- a/bin/mapred +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -bin=`dirname "$0"` -bin=`cd "$bin"; pwd` - -. $bin/mapred-config.sh - -function print_usage(){ - echo "Usage: mapred [--config confdir] COMMAND" - echo " where COMMAND is one of:" - echo " mradmin run a Map-Reduce admin client" - echo " jobtracker run the MapReduce job Tracker node" - echo " tasktracker run a MapReduce task Tracker node" - echo " pipes run a Pipes job" - echo " job manipulate MapReduce jobs" - echo " queue get information regarding JobQueues" - echo "" - echo "Most commands print help when invoked w/o parameters." -} - -if [ $# = 0 ]; then - print_usage - exit -fi - -COMMAND=$1 -shift - -if [ "$COMMAND" = "mradmin" ] ; then - CLASS=org.apache.hadoop.mapred.tools.MRAdmin - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" -elif [ "$COMMAND" = "jobtracker" ] ; then - CLASS=org.apache.hadoop.mapred.JobTracker - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_JOBTRACKER_OPTS" -elif [ "$COMMAND" = "tasktracker" ] ; then - CLASS=org.apache.hadoop.mapred.TaskTracker - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_TASKTRACKER_OPTS" -elif [ "$COMMAND" = "job" ] ; then - CLASS=org.apache.hadoop.mapred.JobClient -elif [ "$COMMAND" = "queue" ] ; then - CLASS=org.apache.hadoop.mapred.JobQueueClient -elif [ "$COMMAND" = "pipes" ] ; then - CLASS=org.apache.hadoop.mapred.pipes.Submitter - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" -elif [ "$COMMAND" = "sampler" ] ; then - CLASS=org.apache.hadoop.mapred.lib.InputSampler - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" -else - echo $COMMAND - invalid command - print_usage - exit -fi - -# for developers, add mapred classes to CLASSPATH -if [ -d "$HADOOP_MAPRED_HOME/build/classes" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build/classes -fi -if [ -d "$HADOOP_MAPRED_HOME/build/webapps" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build -fi -if [ -d "$HADOOP_MAPRED_HOME/build/test/classes" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build/test/classes -fi -if [ -d "$HADOOP_MAPRED_HOME/build/tools" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build/tools -fi - -# for releases, add core mapred jar & webapps to CLASSPATH -if [ -d "$HADOOP_MAPRED_HOME/webapps" ]; then - CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME -fi -for f in $HADOOP_MAPRED_HOME/hadoop-mapred-*.jar; do - CLASSPATH=${CLASSPATH}:$f; -done - -# add libs to CLASSPATH -for f in $HADOOP_MAPRED_HOME/lib/*.jar; do - CLASSPATH=${CLASSPATH}:$f; -done - -if $cygwin; then - CLASSPATH=`cygpath -p -w "$CLASSPATH"` -fi -export CLASSPATH -exec "$JAVA" $JAVA_HEAP_MAX $HADOOP_OPTS $CLASS "$@" diff --git a/bin/mapred-config.sh b/bin/mapred-config.sh deleted file mode 100644 index af35dabab3..0000000000 --- a/bin/mapred-config.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# included in all the mapred scripts with source command -# should not be executed directly - -bin=`dirname "$0"` -bin=`cd "$bin"; pwd` - -#TODO: change the env variable when directory structure is changed -export HADOOP_CORE_HOME="${HADOOP_CORE_HOME:-$bin/..}" -#export HADOOP_CORE_HOME="${HADOOP_CORE_HOME:-$bin/../../core}" - -if [ -d "${HADOOP_CORE_HOME}" ]; then - . "$HADOOP_CORE_HOME"/bin/hadoop-config.sh -else - echo "Hadoop core not found." - exit -fi diff --git a/bin/rcc b/bin/rcc index c16667193b..cddfc14235 100755 --- a/bin/rcc +++ b/bin/rcc @@ -72,7 +72,7 @@ IFS= if [ -d "$HADOOP_HOME/webapps" ]; then CLASSPATH=${CLASSPATH}:$HADOOP_HOME fi -for f in $HADOOP_HOME/hadoop-core-*.jar; do +for f in $HADOOP_HOME/hadoop-*.jar; do CLASSPATH=${CLASSPATH}:$f; done diff --git a/bin/start-balancer.sh b/bin/start-balancer.sh deleted file mode 100755 index da00ec1c0d..0000000000 --- a/bin/start-balancer.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin"; pwd` - -. "$bin"/hdfs-config.sh - -# Start balancer daemon. - -"$HADOOP_CORE_HOME"/bin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs start balancer $@ diff --git a/bin/start-dfs.sh b/bin/start-dfs.sh deleted file mode 100755 index d5f7da8d4e..0000000000 --- a/bin/start-dfs.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# Start hadoop dfs daemons. -# Optinally upgrade or rollback dfs state. -# Run this on master node. - -usage="Usage: start-dfs.sh [-upgrade|-rollback]" - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin"; pwd` - -. "$bin"/hdfs-config.sh - -# get arguments -if [ $# -ge 1 ]; then - nameStartOpt=$1 - shift - case $nameStartOpt in - (-upgrade) - ;; - (-rollback) - dataStartOpt=$nameStartOpt - ;; - (*) - echo $usage - exit 1 - ;; - esac -fi - -# start dfs daemons -# start namenode after datanodes, to minimize time namenode is up w/o data -# note: datanodes will log connection errors until namenode starts -"$HADOOP_CORE_HOME"/bin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs start namenode $nameStartOpt -"$HADOOP_CORE_HOME"/bin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs start datanode $dataStartOpt -"$HADOOP_CORE_HOME"/bin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --hosts masters --script "$bin"/hdfs start secondarynamenode diff --git a/bin/start-mapred.sh b/bin/start-mapred.sh deleted file mode 100755 index 6dc55fe303..0000000000 --- a/bin/start-mapred.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# Start hadoop map reduce daemons. Run this on master node. - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin"; pwd` - -. $bin/mapred-config.sh - -# start mapred daemons -# start jobtracker first to minimize connection errors at startup -"$HADOOP_CORE_HOME"/bin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script "$bin"/mapred start jobtracker -"$HADOOP_CORE_HOME"/bin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/mapred start tasktracker \ No newline at end of file diff --git a/bin/stop-balancer.sh b/bin/stop-balancer.sh deleted file mode 100755 index 7b6326e2ba..0000000000 --- a/bin/stop-balancer.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin"; pwd` - -. "$bin"/hdfs-config.sh - -# Stop balancer daemon. -# Run this on the machine where the balancer is running - -"$HADOOP_CORE_HOME"/bin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs stop balancer diff --git a/bin/stop-dfs.sh b/bin/stop-dfs.sh deleted file mode 100755 index 6fc1f68bd4..0000000000 --- a/bin/stop-dfs.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# Stop hadoop DFS daemons. Run this on master node. - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin"; pwd` - -. "$bin"/hdfs-config.sh - -"$HADOOP_CORE_HOME"/bin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs stop namenode -"$HADOOP_CORE_HOME"/bin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs stop datanode -"$HADOOP_CORE_HOME"/bin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --hosts masters --script "$bin"/hdfs stop secondarynamenode \ No newline at end of file diff --git a/bin/stop-mapred.sh b/bin/stop-mapred.sh deleted file mode 100755 index e611aedaf2..0000000000 --- a/bin/stop-mapred.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# Stop hadoop map reduce daemons. Run this on master node. - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin"; pwd` - -. $bin/mapred-config.sh - -"$HADOOP_CORE_HOME"/bin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script "$bin"/mapred stop jobtracker -"$HADOOP_CORE_HOME"/bin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/mapred stop tasktracker \ No newline at end of file diff --git a/conf/capacity-scheduler.xml.template b/conf/capacity-scheduler.xml.template deleted file mode 100644 index c6347a1a79..0000000000 --- a/conf/capacity-scheduler.xml.template +++ /dev/null @@ -1,126 +0,0 @@ - - - - - - - - - - - mapred.capacity-scheduler.queue.default.capacity - 100 - Percentage of the number of slots in the cluster that are - to be available for jobs in this queue. - - - - - mapred.capacity-scheduler.queue.default.supports-priority - false - If true, priorities of jobs will be taken into - account in scheduling decisions. - - - - - mapred.capacity-scheduler.queue.default.minimum-user-limit-percent - 100 - Each queue enforces a limit on the percentage of resources - allocated to a user at any given time, if there is competition for them. - This user limit can vary between a minimum and maximum value. The former - depends on the number of users who have submitted jobs, and the latter is - set to this property value. For example, suppose the value of this - property is 25. If two users have submitted jobs to a queue, no single - user can use more than 50% of the queue resources. If a third user submits - a job, no single user can use more than 33% of the queue resources. With 4 - or more users, no user can use more than 25% of the queue's resources. A - value of 100 implies no user limits are imposed. - - - - mapred.capacity-scheduler.queue.default.maximum-initialized-jobs-per-user - 2 - The maximum number of jobs to be pre-initialized for a user - of the job queue. - - - - - - - - mapred.capacity-scheduler.default-supports-priority - false - If true, priorities of jobs will be taken into - account in scheduling decisions by default in a job queue. - - - - - mapred.capacity-scheduler.task.default-pmem-percentage-in-vmem - -1 - A percentage (float) of the default VM limit for jobs - (mapred.task.default.maxvm). This is the default RAM task-limit - associated with a task. Unless overridden by a job's setting, this - number defines the RAM task-limit. - - If this property is missing, or set to an invalid value, scheduling - based on physical memory, RAM, is disabled. - - - - - mapred.capacity-scheduler.task.limit.maxpmem - -1 - Configuration that provides an upper limit on the maximum - physical memory that can be specified by a job. The job configuration - mapred.task.maxpmem should be less than this value. If not, the job will - be rejected by the scheduler. - - If it is set to -1, scheduler will not consider physical memory for - scheduling even if virtual memory based scheduling is enabled(by setting - valid values for both mapred.task.default.maxvmem and - mapred.task.limit.maxvmem). - - - - - mapred.capacity-scheduler.default-minimum-user-limit-percent - 100 - The percentage of the resources limited to a particular user - for the job queue at any given point of time by default. - - - - - mapred.capacity-scheduler.default-maximum-initialized-jobs-per-user - 2 - The maximum number of jobs to be pre-initialized for a user - of the job queue. - - - - - - - mapred.capacity-scheduler.init-poll-interval - 5000 - The amount of time in miliseconds which is used to poll - the job queues for jobs to initialize. - - - - mapred.capacity-scheduler.init-worker-threads - 5 - Number of worker threads which would be used by - Initialization poller to initialize jobs in a set of queue. - If number mentioned in property is equal to number of job queues - then a single thread would initialize jobs in a queue. If lesser - then a thread would get a set of queues assigned. If the number - is greater then number of threads would be equal to number of - job queues. - - - - diff --git a/conf/hadoop-policy.xml.template b/conf/hadoop-policy.xml.template index ef48f2bbed..5727b13313 100644 --- a/conf/hadoop-policy.xml.template +++ b/conf/hadoop-policy.xml.template @@ -94,4 +94,13 @@ A special value of "*" means all users are allowed. + + security.admin.operations.protocol.acl + * + ACL for AdminOperationsProtocol, used by the mradmins commands + to refresh queues and nodes at JobTracker. The ACL is a comma-separated list of + user and group names. The user and group list is separated by a blank. + For e.g. "alice,bob users,wheel". A special value of "*" means all users are + allowed. + diff --git a/conf/hdfs-site.xml.template b/conf/hdfs-site.xml.template deleted file mode 100644 index 970c8fe0e8..0000000000 --- a/conf/hdfs-site.xml.template +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - diff --git a/conf/log4j.properties b/conf/log4j.properties index 2783524fef..75b513737d 100644 --- a/conf/log4j.properties +++ b/conf/log4j.properties @@ -3,6 +3,16 @@ hadoop.root.logger=INFO,console hadoop.log.dir=. hadoop.log.file=hadoop.log +# +# Job Summary Appender +# +# Use following logger to send summary to separate file defined by +# hadoop.mapreduce.jobsummary.log.file rolled daily: +# hadoop.mapreduce.jobsummary.logger=INFO,JSA +# +hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger} +hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log + # Define the root logger to the system property "hadoop.root.logger". log4j.rootLogger=${hadoop.root.logger}, EventCounter @@ -45,6 +55,7 @@ log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: #Default values hadoop.tasklog.taskid=null +hadoop.tasklog.iscleanup=false hadoop.tasklog.noKeepSplits=4 hadoop.tasklog.totalLogFileSize=100 hadoop.tasklog.purgeLogSplits=true @@ -52,12 +63,12 @@ hadoop.tasklog.logsRetainHours=12 log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender log4j.appender.TLA.taskId=${hadoop.tasklog.taskid} +log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup} log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize} log4j.appender.TLA.layout=org.apache.log4j.PatternLayout log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n - # #Security appender # @@ -105,3 +116,34 @@ log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR # Sends counts of logging messages at different severity levels to Hadoop Metrics. # log4j.appender.EventCounter=org.apache.hadoop.metrics.jvm.EventCounter + +# +# Job Summary Appender +# +log4j.appender.JSA=org.apache.log4j.DailyRollingFileAppender +log4j.appender.JSA.File=${hadoop.log.dir}/${hadoop.mapreduce.jobsummary.log.file} +log4j.appender.JSA.layout=org.apache.log4j.PatternLayout +log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n +log4j.appender.JSA.DatePattern=.yyyy-MM-dd +log4j.logger.org.apache.hadoop.mapred.JobInProgress$JobSummary=${hadoop.mapreduce.jobsummary.logger} +log4j.additivity.org.apache.hadoop.mapred.JobInProgress$JobSummary=false + +# +# MapReduce Audit Log Appender +# + +# Set the MapReduce audit log filename +#hadoop.mapreduce.audit.log.file=hadoop-mapreduce.audit.log + +# Appender for AuditLogger. +# Requires the following system properties to be set +# - hadoop.log.dir (Hadoop Log directory) +# - hadoop.mapreduce.audit.log.file (MapReduce audit log filename) + +#log4j.logger.org.apache.hadoop.mapred.AuditLogger=INFO,MRAUDIT +#log4j.additivity.org.apache.hadoop.mapred.AuditLogger=false +#log4j.appender.MRAUDIT=org.apache.log4j.DailyRollingFileAppender +#log4j.appender.MRAUDIT.File=${hadoop.log.dir}/${hadoop.mapreduce.audit.log.file} +#log4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd +#log4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout +#log4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n diff --git a/conf/mapred-queue-acls.xml.template b/conf/mapred-queue-acls.xml.template deleted file mode 100644 index 04d118b589..0000000000 --- a/conf/mapred-queue-acls.xml.template +++ /dev/null @@ -1,31 +0,0 @@ - - - - - - - - - mapred.queue.default.acl-submit-job - * - Comma separated list of user and group names that are allowed - to submit jobs to the 'default' queue. The user list and the group list - are separated by a blank. For e.g. alice,bob group1,group2. - If set to the special value '*', it means all users are allowed to - submit jobs. - - - - - mapred.queue.default.acl-administer-jobs - * - Comma separated list of user and group names that are allowed - to delete jobs or modify job's priority for jobs not owned by the current - user in the 'default' queue. The user list and the group list - are separated by a blank. For e.g. alice,bob group1,group2. - If set to the special value '*', it means all users are allowed to do - this operation. - - - - diff --git a/conf/mapred-site.xml.template b/conf/mapred-site.xml.template deleted file mode 100644 index 970c8fe0e8..0000000000 --- a/conf/mapred-site.xml.template +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - diff --git a/conf/taskcontroller.cfg b/conf/taskcontroller.cfg deleted file mode 100644 index fb4cced666..0000000000 --- a/conf/taskcontroller.cfg +++ /dev/null @@ -1,3 +0,0 @@ -mapred.local.dir=#configured value of hadoop.tmp.dir it can be a list of paths comma seperated -hadoop.pid.dir=#configured HADOOP_PID_DIR -hadoop.indent.str=#configured HADOOP_IDENT_STR