HADOOP-7655. Provide a small validation script that smoke tests the installed cluster. Contributed by Arpit Gupta.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1183539 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
57f99c2b02
commit
5f9eab403c
@ -154,6 +154,9 @@ Release 0.23.0 - Unreleased
|
|||||||
|
|
||||||
IMPROVEMENTS
|
IMPROVEMENTS
|
||||||
|
|
||||||
|
HADOOP-7655. Provide a small validation script that smoke tests the installed
|
||||||
|
cluster. (Arpit Gupta via mattf)
|
||||||
|
|
||||||
HADOOP-7042. Updates to test-patch.sh to include failed test names and
|
HADOOP-7042. Updates to test-patch.sh to include failed test names and
|
||||||
improve other messaging. (nigel)
|
improve other messaging. (nigel)
|
||||||
|
|
||||||
|
@ -0,0 +1,181 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
# contributor license agreements. See the NOTICE file distributed with
|
||||||
|
# this work for additional information regarding copyright ownership.
|
||||||
|
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
# (the "License"); you may not use this file except in compliance with
|
||||||
|
# the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
###############################################################################
|
||||||
|
# Run the following jobs to validate a hadoop cluster
|
||||||
|
## teragen
|
||||||
|
## terasort
|
||||||
|
## teravalidate
|
||||||
|
# If they all pass 0 will be returned and 1 otherwise
|
||||||
|
# The test will work for both secure and unsecure deploys. If the kerberos-realm
|
||||||
|
# is passed we will assume that the deploy is secure and proceed with a kinit before
|
||||||
|
# running the validation jobs.
|
||||||
|
################################################################################
|
||||||
|
|
||||||
|
bin=`dirname "$0"`
|
||||||
|
bin=`cd "$bin"; pwd`
|
||||||
|
|
||||||
|
. "$bin"/../libexec/hadoop-config.sh
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
echo "
|
||||||
|
usage: $0 <parameters>
|
||||||
|
|
||||||
|
Optional parameters:
|
||||||
|
-h Display this message
|
||||||
|
--user=hdfs
|
||||||
|
--user_keytab=/home/hdfs/hdfs.keytab
|
||||||
|
--kerberos-realm=KERBEROS.EXAMPLE.COM Set Kerberos realm
|
||||||
|
"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
OPTS=$(getopt \
|
||||||
|
-n $0 \
|
||||||
|
-o '' \
|
||||||
|
-l 'user:' \
|
||||||
|
-l 'user-keytab:' \
|
||||||
|
-l 'kerberos-realm:' \
|
||||||
|
-o 'h' \
|
||||||
|
-- "$@")
|
||||||
|
|
||||||
|
if [ $? != 0 ] ; then
|
||||||
|
usage
|
||||||
|
fi
|
||||||
|
|
||||||
|
eval set -- "${OPTS}"
|
||||||
|
while true ; do
|
||||||
|
case "$1" in
|
||||||
|
--user)
|
||||||
|
TEST_USER=$2; shift 2
|
||||||
|
AUTOMATED=1
|
||||||
|
;;
|
||||||
|
--user-keytab)
|
||||||
|
USER_KEYTAB_FILE=$2; shift 2
|
||||||
|
AUTOMATED=1
|
||||||
|
;;
|
||||||
|
--kerberos-realm)
|
||||||
|
KERBEROS_REALM=$2; shift 2
|
||||||
|
AUTOMATED=1
|
||||||
|
;;
|
||||||
|
--)
|
||||||
|
shift ; break
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Unknown option: $1"
|
||||||
|
usage
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
#set the hadoop command and the path to the hadoop examples jar
|
||||||
|
HADOOP_CMD="${HADOOP_PREFIX}/bin/hadoop --config $HADOOP_CONF_DIR"
|
||||||
|
|
||||||
|
#find the hadoop examples jar
|
||||||
|
HADOOP_EXAMPLES_JAR=''
|
||||||
|
|
||||||
|
#find under HADOOP_PREFIX (tar ball install)
|
||||||
|
HADOOP_EXAMPLES_JAR=`find ${HADOOP_PREFIX} -name 'hadoop-examples-*.jar' | head -n1`
|
||||||
|
|
||||||
|
#if its not found look under /usr/share/hadoop (rpm/deb installs)
|
||||||
|
if [ "$HADOOP_EXAMPLES_JAR" == '' ]
|
||||||
|
then
|
||||||
|
HADOOP_EXAMPLES_JAR=`find /usr/share/hadoop -name 'hadoop-examples-*.jar' | head -n1`
|
||||||
|
fi
|
||||||
|
|
||||||
|
#if it is still empty then dont run the tests
|
||||||
|
if [ "$HADOOP_EXAMPLES_JAR" == '' ]
|
||||||
|
then
|
||||||
|
echo "Did not find hadoop-examples-*.jar under '${HADOOP_PREFIX} or '/usr/share/hadoop'"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# do a kinit if secure
|
||||||
|
if [ "${KERBEROS_REALM}" != "" ]; then
|
||||||
|
# Determine kerberos location base on Linux distro.
|
||||||
|
if [ -e /etc/lsb-release ]; then
|
||||||
|
KERBEROS_BIN=/usr/bin
|
||||||
|
else
|
||||||
|
KERBEROS_BIN=/usr/kerberos/bin
|
||||||
|
fi
|
||||||
|
kinit_cmd="su -c '${KERBEROS_BIN}/kinit -kt ${USER_KEYTAB_FILE} ${TEST_USER}' ${TEST_USER}"
|
||||||
|
echo $kinit_cmd
|
||||||
|
eval $kinit_cmd
|
||||||
|
if [ $? -ne 0 ]
|
||||||
|
then
|
||||||
|
echo "kinit command did not run successfully."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
|
||||||
|
#dir where to store the data on hdfs. The data is relative of the users home dir on hdfs.
|
||||||
|
PARENT_DIR="validate_deploy_`date +%s`"
|
||||||
|
TERA_GEN_OUTPUT_DIR="${PARENT_DIR}/tera_gen_data"
|
||||||
|
TERA_SORT_OUTPUT_DIR="${PARENT_DIR}/tera_sort_data"
|
||||||
|
TERA_VALIDATE_OUTPUT_DIR="${PARENT_DIR}/tera_validate_data"
|
||||||
|
#tera gen cmd
|
||||||
|
TERA_GEN_CMD="su -c '$HADOOP_CMD jar $HADOOP_EXAMPLES_JAR teragen 10000 $TERA_GEN_OUTPUT_DIR' $TEST_USER"
|
||||||
|
|
||||||
|
#tera sort cmd
|
||||||
|
TERA_SORT_CMD="su -c '$HADOOP_CMD jar $HADOOP_EXAMPLES_JAR terasort $TERA_GEN_OUTPUT_DIR $TERA_SORT_OUTPUT_DIR' $TEST_USER"
|
||||||
|
|
||||||
|
#tera validate cmd
|
||||||
|
TERA_VALIDATE_CMD="su -c '$HADOOP_CMD jar $HADOOP_EXAMPLES_JAR teravalidate $TERA_SORT_OUTPUT_DIR $TERA_VALIDATE_OUTPUT_DIR' $TEST_USER"
|
||||||
|
|
||||||
|
echo "Starting teragen...."
|
||||||
|
|
||||||
|
#run tera gen
|
||||||
|
echo $TERA_GEN_CMD
|
||||||
|
eval $TERA_GEN_CMD
|
||||||
|
if [ $? -ne 0 ]; then
|
||||||
|
echo "tera gen failed."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Teragen passed starting terasort...."
|
||||||
|
|
||||||
|
|
||||||
|
#run tera sort
|
||||||
|
echo $TERA_SORT_CMD
|
||||||
|
eval $TERA_SORT_CMD
|
||||||
|
if [ $? -ne 0 ]; then
|
||||||
|
echo "tera sort failed."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Terasort passed starting teravalidate...."
|
||||||
|
|
||||||
|
#run tera validate
|
||||||
|
echo $TERA_VALIDATE_CMD
|
||||||
|
eval $TERA_VALIDATE_CMD
|
||||||
|
if [ $? -ne 0 ]; then
|
||||||
|
echo "tera validate failed."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "teragen, terasort, teravalidate passed."
|
||||||
|
echo "Cleaning the data created by tests: $PARENT_DIR"
|
||||||
|
|
||||||
|
CLEANUP_CMD="su -c '$HADOOP_CMD dfs -rmr -skipTrash $PARENT_DIR' $TEST_USER"
|
||||||
|
echo $CLEANUP_CMD
|
||||||
|
eval $CLEANUP_CMD
|
||||||
|
|
||||||
|
exit 0
|
Loading…
Reference in New Issue
Block a user