diff --git a/CHANGES.txt b/CHANGES.txt
index d3c2feb29d..d04fbe5e5c 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -935,6 +935,8 @@ Release 0.21.0 - Unreleased
HADOOP-6738. Move cluster_setup.xml from MapReduce to Common.
(Tom White via tomwhite)
+ HADOOP-6794. Move configuration and script files post split. (tomwhite)
+
OPTIMIZATIONS
HADOOP-5595. NameNode does not need to run a replicator to choose a
diff --git a/bin/hadoop b/bin/hadoop
index 6d34964ea2..10509037cf 100755
--- a/bin/hadoop
+++ b/bin/hadoop
@@ -53,7 +53,9 @@ case $COMMAND in
echo ""
#try to locate hdfs and if present, delegate to it.
if [ -f "${HADOOP_HDFS_HOME}"/bin/hdfs ]; then
- exec "${HADOOP_HDFS_HOME}"/bin/hdfs $*
+ exec "${HADOOP_HDFS_HOME}"/bin/hdfs $*
+ elif [ -f "${HADOOP_HOME}"/bin/hdfs ]; then
+ exec "${HADOOP_HOME}"/bin/hdfs $*
else
echo "HDFS not found."
exit
@@ -67,7 +69,9 @@ case $COMMAND in
echo ""
#try to locate mapred and if present, delegate to it.
if [ -f "${HADOOP_MAPRED_HOME}"/bin/mapred ]; then
- exec "${HADOOP_MAPRED_HOME}"/bin/mapred $*
+ exec "${HADOOP_MAPRED_HOME}"/bin/mapred $*
+ elif [ -f "${HADOOP_HOME}"/bin/mapred ]; then
+ exec "${HADOOP_HOME}"/bin/mapred $*
else
echo "MAPRED not found."
exit
diff --git a/bin/hadoop-config.sh b/bin/hadoop-config.sh
index aabeffefcf..e821ed7575 100644
--- a/bin/hadoop-config.sh
+++ b/bin/hadoop-config.sh
@@ -31,17 +31,17 @@ while [ -h "$this" ]; do
done
# convert relative path to absolute path
-bin=`dirname "$this"`
+common_bin=`dirname "$this"`
script=`basename "$this"`
-bin=`cd "$bin"; pwd`
-this="$bin/$script"
+common_bin=`cd "$common_bin"; pwd`
+this="$common_bin/$script"
# the root of the Hadoop installation
#TODO: change the env variable when dir structure is changed
export HADOOP_HOME=`dirname "$this"`/..
-export HADOOP_CORE_HOME="${HADOOP_HOME}"
+export HADOOP_COMMON_HOME="${HADOOP_HOME}"
#export HADOOP_HOME=`dirname "$this"`/../..
-#export HADOOP_CORE_HOME="${HADOOP_CORE_HOME:-`dirname "$this"`/..}"
+#export HADOOP_COMMON_HOME="${HADOOP_COMMON_HOME:-`dirname "$this"`/..}"
#check to see if the conf dir is given as an optional argument
if [ $# -gt 1 ]
@@ -106,54 +106,54 @@ CLASSPATH="${HADOOP_CONF_DIR}"
CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar
# for developers, add Hadoop classes to CLASSPATH
-if [ -d "$HADOOP_CORE_HOME/build/classes" ]; then
- CLASSPATH=${CLASSPATH}:$HADOOP_CORE_HOME/build/classes
+if [ -d "$HADOOP_COMMON_HOME/build/classes" ]; then
+ CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/build/classes
fi
-if [ -d "$HADOOP_CORE_HOME/build/webapps" ]; then
- CLASSPATH=${CLASSPATH}:$HADOOP_CORE_HOME/build
+if [ -d "$HADOOP_COMMON_HOME/build/webapps" ]; then
+ CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/build
fi
-if [ -d "$HADOOP_CORE_HOME/build/test/classes" ]; then
- CLASSPATH=${CLASSPATH}:$HADOOP_CORE_HOME/build/test/classes
+if [ -d "$HADOOP_COMMON_HOME/build/test/classes" ]; then
+ CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/build/test/classes
fi
-if [ -d "$HADOOP_CORE_HOME/build/test/core/classes" ]; then
- CLASSPATH=${CLASSPATH}:$HADOOP_CORE_HOME/build/test/core/classes
+if [ -d "$HADOOP_COMMON_HOME/build/test/core/classes" ]; then
+ CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME/build/test/core/classes
fi
# so that filenames w/ spaces are handled correctly in loops below
IFS=
# for releases, add core hadoop jar & webapps to CLASSPATH
-if [ -d "$HADOOP_CORE_HOME/webapps" ]; then
- CLASSPATH=${CLASSPATH}:$HADOOP_CORE_HOME
+if [ -d "$HADOOP_COMMON_HOME/webapps" ]; then
+ CLASSPATH=${CLASSPATH}:$HADOOP_COMMON_HOME
fi
-for f in $HADOOP_CORE_HOME/hadoop-core-*.jar; do
+for f in $HADOOP_COMMON_HOME/hadoop-*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
# add libs to CLASSPATH
-for f in $HADOOP_CORE_HOME/lib/*.jar; do
+for f in $HADOOP_COMMON_HOME/lib/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
-if [ -d "$HADOOP_CORE_HOME/build/ivy/lib/Hadoop-Core/common" ]; then
-for f in $HADOOP_CORE_HOME/build/ivy/lib/Hadoop-Core/common/*.jar; do
+if [ -d "$HADOOP_COMMON_HOME/build/ivy/lib/Hadoop-Common/common" ]; then
+for f in $HADOOP_COMMON_HOME/build/ivy/lib/Hadoop-Common/common/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
fi
-if [ -d "$HADOOP_CORE_HOME/build/ivy/lib/Hadoop-Hdfs/common" ]; then
-for f in $HADOOP_CORE_HOME/build/ivy/lib/Hadoop-Hdfs/common/*.jar; do
+if [ -d "$HADOOP_COMMON_HOME/build/ivy/lib/Hadoop-Hdfs/common" ]; then
+for f in $HADOOP_COMMON_HOME/build/ivy/lib/Hadoop-Hdfs/common/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
fi
-if [ -d "$HADOOP_CORE_HOME/build/ivy/lib/Hadoop/common" ]; then
-for f in $HADOOP_CORE_HOME/build/ivy/lib/Hadoop/common/*.jar; do
+if [ -d "$HADOOP_COMMON_HOME/build/ivy/lib/Hadoop/common" ]; then
+for f in $HADOOP_COMMON_HOME/build/ivy/lib/Hadoop/common/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
fi
-for f in $HADOOP_CORE_HOME/lib/jsp-2.1/*.jar; do
+for f in $HADOOP_COMMON_HOME/lib/jsp-2.1/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
@@ -180,23 +180,23 @@ unset IFS
# cygwin path translation
if $cygwin; then
- HADOOP_CORE_HOME=`cygpath -w "$HADOOP_CORE_HOME"`
+ HADOOP_COMMON_HOME=`cygpath -w "$HADOOP_COMMON_HOME"`
HADOOP_LOG_DIR=`cygpath -w "$HADOOP_LOG_DIR"`
fi
# setup 'java.library.path' for native-hadoop code if necessary
JAVA_LIBRARY_PATH=''
-if [ -d "${HADOOP_CORE_HOME}/build/native" -o -d "${HADOOP_CORE_HOME}/lib/native" ]; then
+if [ -d "${HADOOP_COMMON_HOME}/build/native" -o -d "${HADOOP_COMMON_HOME}/lib/native" ]; then
JAVA_PLATFORM=`CLASSPATH=${CLASSPATH} ${JAVA} -Xmx32m ${HADOOP_JAVA_PLATFORM_OPTS} org.apache.hadoop.util.PlatformName | sed -e "s/ /_/g"`
- if [ -d "$HADOOP_CORE_HOME/build/native" ]; then
- JAVA_LIBRARY_PATH=${HADOOP_CORE_HOME}/build/native/${JAVA_PLATFORM}/lib
+ if [ -d "$HADOOP_COMMON_HOME/build/native" ]; then
+ JAVA_LIBRARY_PATH=${HADOOP_COMMON_HOME}/build/native/${JAVA_PLATFORM}/lib
fi
- if [ -d "${HADOOP_CORE_HOME}/lib/native" ]; then
+ if [ -d "${HADOOP_COMMON_HOME}/lib/native" ]; then
if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
- JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:${HADOOP_CORE_HOME}/lib/native/${JAVA_PLATFORM}
+ JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:${HADOOP_COMMON_HOME}/lib/native/${JAVA_PLATFORM}
else
- JAVA_LIBRARY_PATH=${HADOOP_CORE_HOME}/lib/native/${JAVA_PLATFORM}
+ JAVA_LIBRARY_PATH=${HADOOP_COMMON_HOME}/lib/native/${JAVA_PLATFORM}
fi
fi
fi
@@ -208,7 +208,7 @@ fi
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR"
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.file=$HADOOP_LOGFILE"
-HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.home.dir=$HADOOP_CORE_HOME"
+HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.home.dir=$HADOOP_COMMON_HOME"
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.id.str=$HADOOP_IDENT_STRING"
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.root.logger=${HADOOP_ROOT_LOGGER:-INFO,console}"
if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
@@ -225,6 +225,15 @@ if [ "$HADOOP_HDFS_HOME" = "" ]; then
fi
if [ -d "${HADOOP_HDFS_HOME}" ]; then
+
+ if [ -d "$HADOOP_HDFS_HOME/webapps" ]; then
+ CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME
+ fi
+
+ if [ -d "${HADOOP_HDFS_HOME}/conf" ]; then
+ CLASSPATH=${CLASSPATH}:${HADOOP_HDFS_HOME}/conf
+ fi
+
for f in $HADOOP_HDFS_HOME/hadoop-hdfs-*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
@@ -253,6 +262,15 @@ if [ "$HADOOP_MAPRED_HOME" = "" ]; then
fi
if [ -d "${HADOOP_MAPRED_HOME}" ]; then
+
+ if [ -d "$HADOOP_MAPRED_HOME/webapps" ]; then
+ CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME
+ fi
+
+ if [ -d "${HADOOP_MAPRED_HOME}/conf" ]; then
+ CLASSPATH=${CLASSPATH}:${HADOOP_MAPRED_HOME}/conf
+ fi
+
for f in $HADOOP_MAPRED_HOME/hadoop-mapred-*.jar; do
CLASSPATH=${CLASSPATH}:$f
done
@@ -284,6 +302,3 @@ if $cygwin; then
fi
-# TODO:remove this when dir structure is changed
-#export HADOOP_HDFS_HOME=$HADOOP_HOME
-#export HADOOP_MAPRED_HOME=$HADOOP_HOME
diff --git a/bin/hdfs b/bin/hdfs
deleted file mode 100755
index 1bc6565cd0..0000000000
--- a/bin/hdfs
+++ /dev/null
@@ -1,113 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-bin=`dirname "$0"`
-bin=`cd "$bin"; pwd`
-
-. "$bin"/hdfs-config.sh
-
-function print_usage(){
- echo "Usage: hdfs [--config confdir] COMMAND"
- echo " where COMMAND is one of:"
- echo " namenode -format format the DFS filesystem"
- echo " secondarynamenode run the DFS secondary namenode"
- echo " namenode run the DFS namenode"
- echo " datanode run a DFS datanode"
- echo " dfsadmin run a DFS admin client"
- echo " fsck run a DFS filesystem checking utility"
- echo " balancer run a cluster balancing utility"
- echo " jmxget get JMX exported values from NameNode or DataNode."
- echo " oiv apply the offline fsimage viewer to an fsimage"
- echo " fetchdt fetch a delegation token from the NameNode"
- echo " Use -help to see options"
- echo ""
- echo "Most commands print help when invoked w/o parameters."
-}
-
-if [ $# = 0 ]; then
- print_usage
- exit
-fi
-
-COMMAND=$1
-shift
-
-if [ "$COMMAND" = "namenode" ] ; then
- CLASS='org.apache.hadoop.hdfs.server.namenode.NameNode'
- HADOOP_OPTS="$HADOOP_OPTS $HADOOP_NAMENODE_OPTS"
-elif [ "$COMMAND" = "secondarynamenode" ] ; then
- CLASS='org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode'
- HADOOP_OPTS="$HADOOP_OPTS $HADOOP_SECONDARYNAMENODE_OPTS"
-elif [ "$COMMAND" = "datanode" ] ; then
- CLASS='org.apache.hadoop.hdfs.server.datanode.DataNode'
- HADOOP_OPTS="$HADOOP_OPTS $HADOOP_DATANODE_OPTS"
-elif [ "$COMMAND" = "dfs" ] ; then
- CLASS=org.apache.hadoop.fs.FsShell
- HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
-elif [ "$COMMAND" = "dfsadmin" ] ; then
- CLASS=org.apache.hadoop.hdfs.tools.DFSAdmin
- HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
-elif [ "$COMMAND" = "fsck" ] ; then
- CLASS=org.apache.hadoop.hdfs.tools.DFSck
- HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
-elif [ "$COMMAND" = "balancer" ] ; then
- CLASS=org.apache.hadoop.hdfs.server.balancer.Balancer
- HADOOP_OPTS="$HADOOP_OPTS $HADOOP_BALANCER_OPTS"
-elif [ "$COMMAND" = "jmxget" ] ; then
- CLASS=org.apache.hadoop.hdfs.tools.JMXGet
-elif [ "$COMMAND" = "oiv" ] ; then
- CLASS=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewer
-elif [ "$COMMAND" = "fetchdt" ] ; then
- CLASS=org.apache.hadoop.hdfs.tools.DelegationTokenFetcher
-else
- echo $COMMAND - invalid command
- print_usage
- exit
-fi
-
-# for developers, add hdfs classes to CLASSPATH
-if [ -d "$HADOOP_HDFS_HOME/build/classes" ]; then
- CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/build/classes
-fi
-if [ -d "$HADOOP_HDFS_HOME/build/webapps" ]; then
- CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/build
-fi
-if [ -d "$HADOOP_HDFS_HOME/build/test/classes" ]; then
- CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/build/test/classes
-fi
-if [ -d "$HADOOP_HDFS_HOME/build/tools" ]; then
- CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME/build/tools
-fi
-
-# for releases, add core hdfs jar & webapps to CLASSPATH
-if [ -d "$HADOOP_HDFS_HOME/webapps" ]; then
- CLASSPATH=${CLASSPATH}:$HADOOP_HDFS_HOME
-fi
-for f in $HADOOP_HDFS_HOME/hadoop-hdfs-*.jar; do
- CLASSPATH=${CLASSPATH}:$f;
-done
-
-# add libs to CLASSPATH
-for f in $HADOOP_HDFS_HOME/lib/*.jar; do
- CLASSPATH=${CLASSPATH}:$f;
-done
-
-if $cygwin; then
- CLASSPATH=`cygpath -p -w "$CLASSPATH"`
-fi
-export CLASSPATH=$CLASSPATH
-exec "$JAVA" $JAVA_HEAP_MAX $HADOOP_OPTS $CLASS "$@"
diff --git a/bin/hdfs-config.sh b/bin/hdfs-config.sh
deleted file mode 100644
index 9f7ad736e9..0000000000
--- a/bin/hdfs-config.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# included in all the hdfs scripts with source command
-# should not be executed directly
-
-bin=`dirname "$0"`
-bin=`cd "$bin"; pwd`
-
-#TODO: change the env variable when directory structure is changed
-export HADOOP_CORE_HOME="${HADOOP_CORE_HOME:-$bin/..}"
-#export HADOOP_CORE_HOME="${HADOOP_CORE_HOME:-$bin/../../core}"
-
-if [ -d "${HADOOP_CORE_HOME}" ]; then
- . "$HADOOP_CORE_HOME"/bin/hadoop-config.sh
-else
- echo "Hadoop core not found."
- exit
-fi
diff --git a/bin/mapred b/bin/mapred
deleted file mode 100755
index c56a19da23..0000000000
--- a/bin/mapred
+++ /dev/null
@@ -1,100 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-bin=`dirname "$0"`
-bin=`cd "$bin"; pwd`
-
-. $bin/mapred-config.sh
-
-function print_usage(){
- echo "Usage: mapred [--config confdir] COMMAND"
- echo " where COMMAND is one of:"
- echo " mradmin run a Map-Reduce admin client"
- echo " jobtracker run the MapReduce job Tracker node"
- echo " tasktracker run a MapReduce task Tracker node"
- echo " pipes run a Pipes job"
- echo " job manipulate MapReduce jobs"
- echo " queue get information regarding JobQueues"
- echo ""
- echo "Most commands print help when invoked w/o parameters."
-}
-
-if [ $# = 0 ]; then
- print_usage
- exit
-fi
-
-COMMAND=$1
-shift
-
-if [ "$COMMAND" = "mradmin" ] ; then
- CLASS=org.apache.hadoop.mapred.tools.MRAdmin
- HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
-elif [ "$COMMAND" = "jobtracker" ] ; then
- CLASS=org.apache.hadoop.mapred.JobTracker
- HADOOP_OPTS="$HADOOP_OPTS $HADOOP_JOBTRACKER_OPTS"
-elif [ "$COMMAND" = "tasktracker" ] ; then
- CLASS=org.apache.hadoop.mapred.TaskTracker
- HADOOP_OPTS="$HADOOP_OPTS $HADOOP_TASKTRACKER_OPTS"
-elif [ "$COMMAND" = "job" ] ; then
- CLASS=org.apache.hadoop.mapred.JobClient
-elif [ "$COMMAND" = "queue" ] ; then
- CLASS=org.apache.hadoop.mapred.JobQueueClient
-elif [ "$COMMAND" = "pipes" ] ; then
- CLASS=org.apache.hadoop.mapred.pipes.Submitter
- HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
-elif [ "$COMMAND" = "sampler" ] ; then
- CLASS=org.apache.hadoop.mapred.lib.InputSampler
- HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
-else
- echo $COMMAND - invalid command
- print_usage
- exit
-fi
-
-# for developers, add mapred classes to CLASSPATH
-if [ -d "$HADOOP_MAPRED_HOME/build/classes" ]; then
- CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build/classes
-fi
-if [ -d "$HADOOP_MAPRED_HOME/build/webapps" ]; then
- CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build
-fi
-if [ -d "$HADOOP_MAPRED_HOME/build/test/classes" ]; then
- CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build/test/classes
-fi
-if [ -d "$HADOOP_MAPRED_HOME/build/tools" ]; then
- CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/build/tools
-fi
-
-# for releases, add core mapred jar & webapps to CLASSPATH
-if [ -d "$HADOOP_MAPRED_HOME/webapps" ]; then
- CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME
-fi
-for f in $HADOOP_MAPRED_HOME/hadoop-mapred-*.jar; do
- CLASSPATH=${CLASSPATH}:$f;
-done
-
-# add libs to CLASSPATH
-for f in $HADOOP_MAPRED_HOME/lib/*.jar; do
- CLASSPATH=${CLASSPATH}:$f;
-done
-
-if $cygwin; then
- CLASSPATH=`cygpath -p -w "$CLASSPATH"`
-fi
-export CLASSPATH
-exec "$JAVA" $JAVA_HEAP_MAX $HADOOP_OPTS $CLASS "$@"
diff --git a/bin/mapred-config.sh b/bin/mapred-config.sh
deleted file mode 100644
index af35dabab3..0000000000
--- a/bin/mapred-config.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# included in all the mapred scripts with source command
-# should not be executed directly
-
-bin=`dirname "$0"`
-bin=`cd "$bin"; pwd`
-
-#TODO: change the env variable when directory structure is changed
-export HADOOP_CORE_HOME="${HADOOP_CORE_HOME:-$bin/..}"
-#export HADOOP_CORE_HOME="${HADOOP_CORE_HOME:-$bin/../../core}"
-
-if [ -d "${HADOOP_CORE_HOME}" ]; then
- . "$HADOOP_CORE_HOME"/bin/hadoop-config.sh
-else
- echo "Hadoop core not found."
- exit
-fi
diff --git a/bin/rcc b/bin/rcc
index c16667193b..cddfc14235 100755
--- a/bin/rcc
+++ b/bin/rcc
@@ -72,7 +72,7 @@ IFS=
if [ -d "$HADOOP_HOME/webapps" ]; then
CLASSPATH=${CLASSPATH}:$HADOOP_HOME
fi
-for f in $HADOOP_HOME/hadoop-core-*.jar; do
+for f in $HADOOP_HOME/hadoop-*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
diff --git a/bin/start-balancer.sh b/bin/start-balancer.sh
deleted file mode 100755
index da00ec1c0d..0000000000
--- a/bin/start-balancer.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-bin=`dirname "${BASH_SOURCE-$0}"`
-bin=`cd "$bin"; pwd`
-
-. "$bin"/hdfs-config.sh
-
-# Start balancer daemon.
-
-"$HADOOP_CORE_HOME"/bin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs start balancer $@
diff --git a/bin/start-dfs.sh b/bin/start-dfs.sh
deleted file mode 100755
index d5f7da8d4e..0000000000
--- a/bin/start-dfs.sh
+++ /dev/null
@@ -1,52 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# Start hadoop dfs daemons.
-# Optinally upgrade or rollback dfs state.
-# Run this on master node.
-
-usage="Usage: start-dfs.sh [-upgrade|-rollback]"
-
-bin=`dirname "${BASH_SOURCE-$0}"`
-bin=`cd "$bin"; pwd`
-
-. "$bin"/hdfs-config.sh
-
-# get arguments
-if [ $# -ge 1 ]; then
- nameStartOpt=$1
- shift
- case $nameStartOpt in
- (-upgrade)
- ;;
- (-rollback)
- dataStartOpt=$nameStartOpt
- ;;
- (*)
- echo $usage
- exit 1
- ;;
- esac
-fi
-
-# start dfs daemons
-# start namenode after datanodes, to minimize time namenode is up w/o data
-# note: datanodes will log connection errors until namenode starts
-"$HADOOP_CORE_HOME"/bin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs start namenode $nameStartOpt
-"$HADOOP_CORE_HOME"/bin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs start datanode $dataStartOpt
-"$HADOOP_CORE_HOME"/bin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --hosts masters --script "$bin"/hdfs start secondarynamenode
diff --git a/bin/start-mapred.sh b/bin/start-mapred.sh
deleted file mode 100755
index 6dc55fe303..0000000000
--- a/bin/start-mapred.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# Start hadoop map reduce daemons. Run this on master node.
-
-bin=`dirname "${BASH_SOURCE-$0}"`
-bin=`cd "$bin"; pwd`
-
-. $bin/mapred-config.sh
-
-# start mapred daemons
-# start jobtracker first to minimize connection errors at startup
-"$HADOOP_CORE_HOME"/bin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script "$bin"/mapred start jobtracker
-"$HADOOP_CORE_HOME"/bin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/mapred start tasktracker
\ No newline at end of file
diff --git a/bin/stop-balancer.sh b/bin/stop-balancer.sh
deleted file mode 100755
index 7b6326e2ba..0000000000
--- a/bin/stop-balancer.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-bin=`dirname "${BASH_SOURCE-$0}"`
-bin=`cd "$bin"; pwd`
-
-. "$bin"/hdfs-config.sh
-
-# Stop balancer daemon.
-# Run this on the machine where the balancer is running
-
-"$HADOOP_CORE_HOME"/bin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs stop balancer
diff --git a/bin/stop-dfs.sh b/bin/stop-dfs.sh
deleted file mode 100755
index 6fc1f68bd4..0000000000
--- a/bin/stop-dfs.sh
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# Stop hadoop DFS daemons. Run this on master node.
-
-bin=`dirname "${BASH_SOURCE-$0}"`
-bin=`cd "$bin"; pwd`
-
-. "$bin"/hdfs-config.sh
-
-"$HADOOP_CORE_HOME"/bin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs stop namenode
-"$HADOOP_CORE_HOME"/bin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs stop datanode
-"$HADOOP_CORE_HOME"/bin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --hosts masters --script "$bin"/hdfs stop secondarynamenode
\ No newline at end of file
diff --git a/bin/stop-mapred.sh b/bin/stop-mapred.sh
deleted file mode 100755
index e611aedaf2..0000000000
--- a/bin/stop-mapred.sh
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# Stop hadoop map reduce daemons. Run this on master node.
-
-bin=`dirname "${BASH_SOURCE-$0}"`
-bin=`cd "$bin"; pwd`
-
-. $bin/mapred-config.sh
-
-"$HADOOP_CORE_HOME"/bin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script "$bin"/mapred stop jobtracker
-"$HADOOP_CORE_HOME"/bin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/mapred stop tasktracker
\ No newline at end of file
diff --git a/conf/capacity-scheduler.xml.template b/conf/capacity-scheduler.xml.template
deleted file mode 100644
index c6347a1a79..0000000000
--- a/conf/capacity-scheduler.xml.template
+++ /dev/null
@@ -1,126 +0,0 @@
-
-
-
-
-
-
-
-
-
-
- mapred.capacity-scheduler.queue.default.capacity
- 100
- Percentage of the number of slots in the cluster that are
- to be available for jobs in this queue.
-
-
-
-
- mapred.capacity-scheduler.queue.default.supports-priority
- false
- If true, priorities of jobs will be taken into
- account in scheduling decisions.
-
-
-
-
- mapred.capacity-scheduler.queue.default.minimum-user-limit-percent
- 100
- Each queue enforces a limit on the percentage of resources
- allocated to a user at any given time, if there is competition for them.
- This user limit can vary between a minimum and maximum value. The former
- depends on the number of users who have submitted jobs, and the latter is
- set to this property value. For example, suppose the value of this
- property is 25. If two users have submitted jobs to a queue, no single
- user can use more than 50% of the queue resources. If a third user submits
- a job, no single user can use more than 33% of the queue resources. With 4
- or more users, no user can use more than 25% of the queue's resources. A
- value of 100 implies no user limits are imposed.
-
-
-
- mapred.capacity-scheduler.queue.default.maximum-initialized-jobs-per-user
- 2
- The maximum number of jobs to be pre-initialized for a user
- of the job queue.
-
-
-
-
-
-
-
- mapred.capacity-scheduler.default-supports-priority
- false
- If true, priorities of jobs will be taken into
- account in scheduling decisions by default in a job queue.
-
-
-
-
- mapred.capacity-scheduler.task.default-pmem-percentage-in-vmem
- -1
- A percentage (float) of the default VM limit for jobs
- (mapred.task.default.maxvm). This is the default RAM task-limit
- associated with a task. Unless overridden by a job's setting, this
- number defines the RAM task-limit.
-
- If this property is missing, or set to an invalid value, scheduling
- based on physical memory, RAM, is disabled.
-
-
-
-
- mapred.capacity-scheduler.task.limit.maxpmem
- -1
- Configuration that provides an upper limit on the maximum
- physical memory that can be specified by a job. The job configuration
- mapred.task.maxpmem should be less than this value. If not, the job will
- be rejected by the scheduler.
-
- If it is set to -1, scheduler will not consider physical memory for
- scheduling even if virtual memory based scheduling is enabled(by setting
- valid values for both mapred.task.default.maxvmem and
- mapred.task.limit.maxvmem).
-
-
-
-
- mapred.capacity-scheduler.default-minimum-user-limit-percent
- 100
- The percentage of the resources limited to a particular user
- for the job queue at any given point of time by default.
-
-
-
-
- mapred.capacity-scheduler.default-maximum-initialized-jobs-per-user
- 2
- The maximum number of jobs to be pre-initialized for a user
- of the job queue.
-
-
-
-
-
-
- mapred.capacity-scheduler.init-poll-interval
- 5000
- The amount of time in miliseconds which is used to poll
- the job queues for jobs to initialize.
-
-
-
- mapred.capacity-scheduler.init-worker-threads
- 5
- Number of worker threads which would be used by
- Initialization poller to initialize jobs in a set of queue.
- If number mentioned in property is equal to number of job queues
- then a single thread would initialize jobs in a queue. If lesser
- then a thread would get a set of queues assigned. If the number
- is greater then number of threads would be equal to number of
- job queues.
-
-
-
-
diff --git a/conf/hadoop-policy.xml.template b/conf/hadoop-policy.xml.template
index ef48f2bbed..5727b13313 100644
--- a/conf/hadoop-policy.xml.template
+++ b/conf/hadoop-policy.xml.template
@@ -94,4 +94,13 @@
A special value of "*" means all users are allowed.
+
+ security.admin.operations.protocol.acl
+ *
+ ACL for AdminOperationsProtocol, used by the mradmins commands
+ to refresh queues and nodes at JobTracker. The ACL is a comma-separated list of
+ user and group names. The user and group list is separated by a blank.
+ For e.g. "alice,bob users,wheel". A special value of "*" means all users are
+ allowed.
+
diff --git a/conf/hdfs-site.xml.template b/conf/hdfs-site.xml.template
deleted file mode 100644
index 970c8fe0e8..0000000000
--- a/conf/hdfs-site.xml.template
+++ /dev/null
@@ -1,8 +0,0 @@
-
-
-
-
-
-
-
-
diff --git a/conf/log4j.properties b/conf/log4j.properties
index 2783524fef..75b513737d 100644
--- a/conf/log4j.properties
+++ b/conf/log4j.properties
@@ -3,6 +3,16 @@ hadoop.root.logger=INFO,console
hadoop.log.dir=.
hadoop.log.file=hadoop.log
+#
+# Job Summary Appender
+#
+# Use following logger to send summary to separate file defined by
+# hadoop.mapreduce.jobsummary.log.file rolled daily:
+# hadoop.mapreduce.jobsummary.logger=INFO,JSA
+#
+hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger}
+hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log
+
# Define the root logger to the system property "hadoop.root.logger".
log4j.rootLogger=${hadoop.root.logger}, EventCounter
@@ -45,6 +55,7 @@ log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}:
#Default values
hadoop.tasklog.taskid=null
+hadoop.tasklog.iscleanup=false
hadoop.tasklog.noKeepSplits=4
hadoop.tasklog.totalLogFileSize=100
hadoop.tasklog.purgeLogSplits=true
@@ -52,12 +63,12 @@ hadoop.tasklog.logsRetainHours=12
log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender
log4j.appender.TLA.taskId=${hadoop.tasklog.taskid}
+log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}
log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}
log4j.appender.TLA.layout=org.apache.log4j.PatternLayout
log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-
#
#Security appender
#
@@ -105,3 +116,34 @@ log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR
# Sends counts of logging messages at different severity levels to Hadoop Metrics.
#
log4j.appender.EventCounter=org.apache.hadoop.metrics.jvm.EventCounter
+
+#
+# Job Summary Appender
+#
+log4j.appender.JSA=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.JSA.File=${hadoop.log.dir}/${hadoop.mapreduce.jobsummary.log.file}
+log4j.appender.JSA.layout=org.apache.log4j.PatternLayout
+log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
+log4j.appender.JSA.DatePattern=.yyyy-MM-dd
+log4j.logger.org.apache.hadoop.mapred.JobInProgress$JobSummary=${hadoop.mapreduce.jobsummary.logger}
+log4j.additivity.org.apache.hadoop.mapred.JobInProgress$JobSummary=false
+
+#
+# MapReduce Audit Log Appender
+#
+
+# Set the MapReduce audit log filename
+#hadoop.mapreduce.audit.log.file=hadoop-mapreduce.audit.log
+
+# Appender for AuditLogger.
+# Requires the following system properties to be set
+# - hadoop.log.dir (Hadoop Log directory)
+# - hadoop.mapreduce.audit.log.file (MapReduce audit log filename)
+
+#log4j.logger.org.apache.hadoop.mapred.AuditLogger=INFO,MRAUDIT
+#log4j.additivity.org.apache.hadoop.mapred.AuditLogger=false
+#log4j.appender.MRAUDIT=org.apache.log4j.DailyRollingFileAppender
+#log4j.appender.MRAUDIT.File=${hadoop.log.dir}/${hadoop.mapreduce.audit.log.file}
+#log4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd
+#log4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout
+#log4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
diff --git a/conf/mapred-queue-acls.xml.template b/conf/mapred-queue-acls.xml.template
deleted file mode 100644
index 04d118b589..0000000000
--- a/conf/mapred-queue-acls.xml.template
+++ /dev/null
@@ -1,31 +0,0 @@
-
-
-
-
-
-
-
-
- mapred.queue.default.acl-submit-job
- *
- Comma separated list of user and group names that are allowed
- to submit jobs to the 'default' queue. The user list and the group list
- are separated by a blank. For e.g. alice,bob group1,group2.
- If set to the special value '*', it means all users are allowed to
- submit jobs.
-
-
-
-
- mapred.queue.default.acl-administer-jobs
- *
- Comma separated list of user and group names that are allowed
- to delete jobs or modify job's priority for jobs not owned by the current
- user in the 'default' queue. The user list and the group list
- are separated by a blank. For e.g. alice,bob group1,group2.
- If set to the special value '*', it means all users are allowed to do
- this operation.
-
-
-
-
diff --git a/conf/mapred-site.xml.template b/conf/mapred-site.xml.template
deleted file mode 100644
index 970c8fe0e8..0000000000
--- a/conf/mapred-site.xml.template
+++ /dev/null
@@ -1,8 +0,0 @@
-
-
-
-
-
-
-
-
diff --git a/conf/taskcontroller.cfg b/conf/taskcontroller.cfg
deleted file mode 100644
index fb4cced666..0000000000
--- a/conf/taskcontroller.cfg
+++ /dev/null
@@ -1,3 +0,0 @@
-mapred.local.dir=#configured value of hadoop.tmp.dir it can be a list of paths comma seperated
-hadoop.pid.dir=#configured HADOOP_PID_DIR
-hadoop.indent.str=#configured HADOOP_IDENT_STR