diff --git a/dev-support/shelldocs.py b/dev-support/shelldocs.py index 25474508c0..fc7601a61e 100755 --- a/dev-support/shelldocs.py +++ b/dev-support/shelldocs.py @@ -17,6 +17,26 @@ import string from optparse import OptionParser +asflicense=''' + +''' + def docstrip(key,string): string=re.sub("^## @%s " % key ,"",string) string=string.lstrip() @@ -220,17 +240,18 @@ def main(): funcdef.addreturn(line) elif line.startswith('function'): funcdef.setname(line) - if options.skipprnorep: - if funcdef.getaudience() == "Private" and \ - funcdef.getreplace() == "No": + if options.skipprnorep and \ + funcdef.getaudience() == "Private" and \ + funcdef.getreplace() == "No": pass - else: - allfuncs.append(funcdef) + else: + allfuncs.append(funcdef) funcdef=ShellFunction() allfuncs=sorted(allfuncs) outfile=open(options.outfile, "w") + outfile.write(asflicense) for line in toc(allfuncs): outfile.write(line) diff --git a/dev-support/test-patch.d/checkstyle.sh b/dev-support/test-patch.d/checkstyle.sh new file mode 100755 index 0000000000..460709e311 --- /dev/null +++ b/dev-support/test-patch.d/checkstyle.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_plugin checkstyle + +CHECKSTYLE_TIMER=0 + +# if it ends in an explicit .sh, then this is shell code. +# if it doesn't have an extension, we assume it is shell code too +function checkstyle_filefilter +{ + local filename=$1 + + if [[ ${filename} =~ \.java$ ]]; then + add_test checkstyle + fi +} + +function checkstyle_preapply +{ + verify_needed_test checkstyle + + if [[ $? == 0 ]]; then + return 0 + fi + + big_console_header "checkstyle plugin: prepatch" + + start_clock + echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}checkstyle.txt" "${MVN}" test checkstyle:checkstyle-aggregate -DskipTests "-D${PROJECT_NAME}PatchProcess" + if [[ $? != 0 ]] ; then + echo "Pre-patch ${PATCH_BRANCH} checkstyle compilation is broken?" + add_jira_table -1 checkstyle "Pre-patch ${PATCH_BRANCH} checkstyle compilation may be broken." + return 1 + fi + + cp -p "${BASEDIR}/target/checkstyle-result.xml" \ + "${PATCH_DIR}/checkstyle-result-${PATCH_BRANCH}.xml" + + # keep track of how much as elapsed for us already + CHECKSTYLE_TIMER=$(stop_clock) + return 0 +} + +function checkstyle_postapply +{ + verify_needed_test checkstyle + + if [[ $? == 0 ]]; then + return 0 + fi + + big_console_header "checkstyle plugin: postpatch" + + start_clock + + # add our previous elapsed to our new timer + # by setting the clock back + offset_clock "${CHECKSTYLE_TIMER}" + + echo_and_redirect "${PATCH_DIR}/patchcheckstyle.txt" "${MVN}" test checkstyle:checkstyle-aggregate -DskipTests "-D${PROJECT_NAME}PatchProcess" + if [[ $? != 0 ]] ; then + echo "Post-patch checkstyle compilation is broken." + add_jira_table -1 checkstyle "Post-patch checkstyle compilation is broken." + return 1 + fi + + cp -p "${BASEDIR}/target/checkstyle-result.xml" \ + "${PATCH_DIR}/checkstyle-result-patch.xml" + + checkstyle_runcomparison + + # shellcheck disable=SC2016 + CHECKSTYLE_POSTPATCH=$(wc -l "${PATCH_DIR}/checkstyle-result-diff.txt" | ${AWK} '{print $1}') + + if [[ ${CHECKSTYLE_POSTPATCH} -gt 0 ]] ; then + + add_jira_table -1 checkstyle "The applied patch generated "\ + "${CHECKSTYLE_POSTPATCH}" \ + " additional checkstyle issues." + add_jira_footer checkstyle "@@BASE@@/checkstyle-result-diff.txt" + + return 1 + fi + add_jira_table +1 checkstyle "There were no new checkstyle issues." + return 0 +} + + +function checkstyle_runcomparison +{ + + python <(cat < master_dict[k]: + print_row(k, master_dict[k], child_errors) + +EOF +) "${PATCH_DIR}/checkstyle-result-${PATCH_BRANCH}.xml" "${PATCH_DIR}/checkstyle-result-patch.xml" > "${PATCH_DIR}/checkstyle-result-diff.txt" + +} diff --git a/dev-support/test-patch.d/shellcheck.sh b/dev-support/test-patch.d/shellcheck.sh new file mode 100755 index 0000000000..b60da36f3c --- /dev/null +++ b/dev-support/test-patch.d/shellcheck.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_plugin shellcheck + +SHELLCHECK_TIMER=0 + +SHELLCHECK=${SHELLCHECK:-$(which shellcheck)} + +SHELLCHECK_SPECIFICFILES="" + +# if it ends in an explicit .sh, then this is shell code. +# if it doesn't have an extension, we assume it is shell code too +function shellcheck_filefilter +{ + local filename=$1 + + if [[ ${filename} =~ \.sh$ ]]; then + add_test shellcheck + SHELLCHECK_SPECIFICFILES="${SHELLCHECK_SPECIFICFILES} ./${filename}" + fi + + if [[ ! ${filename} =~ \. ]]; then + add_test shellcheck + fi +} + +function shellcheck_private_findbash +{ + local i + local value + local list + + while read line; do + value=$(find "${line}" ! -name '*.cmd' -type f \ + | ${GREP} -E -v '(.orig$|.rej$)') + list="${list} ${value}" + done < <(find . -type d -name bin -o -type d -name sbin -o -type d -name libexec -o -type d -name shellprofile.d) + # shellcheck disable=SC2086 + echo ${list} ${SHELLCHECK_SPECIFICFILES} | tr ' ' '\n' | sort -u +} + +function shellcheck_preapply +{ + local i + + verify_needed_test shellcheck + if [[ $? == 0 ]]; then + return 0 + fi + + big_console_header "shellcheck plugin: prepatch" + + if [[ -z "${SHELLCHECK}" ]]; then + hadoop_error "shellcheck is not available." + fi + + start_clock + + # shellcheck disable=SC2016 + SHELLCHECK_VERSION=$(shellcheck --version | ${GREP} version: | ${AWK} '{print $NF}') + + echo "Running shellcheck against all identifiable shell scripts" + pushd "${BASEDIR}" >/dev/null + for i in $(shellcheck_private_findbash); do + if [[ -f ${i} ]]; then + ${SHELLCHECK} -f gcc "${i}" >> "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" + fi + done + popd > /dev/null + # keep track of how much as elapsed for us already + SHELLCHECK_TIMER=$(stop_clock) + return 0 +} + +function shellcheck_postapply +{ + local i + + verify_needed_test shellcheck + if [[ $? == 0 ]]; then + return 0 + fi + + big_console_header "shellcheck plugin: postpatch" + + if [[ -z "${SHELLCHECK}" ]]; then + hadoop_error "shellcheck is not available." + fi + + start_clock + + # add our previous elapsed to our new timer + # by setting the clock back + offset_clock "${SHELLCHECK_TIMER}" + + echo "Running shellcheck against all identifiable shell scripts" + # we re-check this in case one has been added + for i in $(shellcheck_private_findbash); do + ${SHELLCHECK} -f gcc "${i}" >> "${PATCH_DIR}/patchshellcheck-result.txt" + done + + # shellcheck disable=SC2016 + numPrepatch=$(wc -l "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" | ${AWK} '{print $1}') + # shellcheck disable=SC2016 + numPostpatch=$(wc -l "${PATCH_DIR}/patchshellcheck-result.txt" | ${AWK} '{print $1}') + + ${DIFF} -u "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" \ + "${PATCH_DIR}/patchshellcheck-result.txt" \ + | ${GREP} '^+\.' \ + > "${PATCH_DIR}/diffpatchshellcheck.txt" + + # shellcheck disable=SC2016 + diffPostpatch=$(wc -l "${PATCH_DIR}/diffpatchshellcheck.txt" | ${AWK} '{print $1}') + + if [[ ${diffPostpatch} -gt 0 ]] ; then + add_jira_table -1 shellcheck "The applied patch generated "\ + "${diffPostpatch} new shellcheck (v${SHELLCHECK_VERSION}) issues (total was ${numPrepatch}, now ${numPostpatch})." + add_jira_footer shellcheck "@@BASE@@/diffpatchshellcheck.txt" + return 1 + fi + + add_jira_table +1 shellcheck "There were no new shellcheck (v${SHELLCHECK_VERSION}) issues." + return 0 +} diff --git a/dev-support/test-patch.d/whitespace.sh b/dev-support/test-patch.d/whitespace.sh new file mode 100755 index 0000000000..deac654da8 --- /dev/null +++ b/dev-support/test-patch.d/whitespace.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_plugin whitespace + +function whitespace_preapply +{ + local count + + big_console_header "Checking for whitespace at the end of lines" + start_clock + + ${GREP} '^+' "${PATCH_DIR}/patch" | ${GREP} '[[:blank:]]$' > "${PATCH_DIR}/whitespace.txt" + + # shellcheck disable=SC2016 + count=$(wc -l "${PATCH_DIR}/whitespace.txt" | ${AWK} '{print $1}') + + if [[ ${count} -gt 0 ]]; then + add_jira_table -1 whitespace "The patch has ${count}"\ + " line(s) that end in whitespace." + add_jira_footer whitespace "@@BASE@@/whitespace.txt" + return 1 + fi + + add_jira_table +1 whitespace "The patch has no lines that end in whitespace." + return 0 +} diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh index 574a4fd7e8..6e8679e49c 100755 --- a/dev-support/test-patch.sh +++ b/dev-support/test-patch.sh @@ -1,728 +1,1740 @@ #!/usr/bin/env bash -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. - -#set -x - -### Setup some variables. ### BUILD_URL is set by Hudson if it is run by patch process -### Read variables from properties file -bindir=$(dirname $0) -# Defaults -if [ -z "$MAVEN_HOME" ]; then - MVN=mvn -else - MVN=$MAVEN_HOME/bin/mvn -fi +this="${BASH_SOURCE-$0}" +BINDIR=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) +CWD=$(pwd) +USER_PARAMS=("$@") +GLOBALTIMER=$(date +"%s") -PROJECT_NAME=Hadoop -JENKINS=false -PATCH_DIR=/tmp -SUPPORT_DIR=/tmp -BASEDIR=$(pwd) -BUILD_NATIVE=true -PS=${PS:-ps} -AWK=${AWK:-awk} -WGET=${WGET:-wget} -GIT=${GIT:-git} -GREP=${GREP:-grep} -PATCH=${PATCH:-patch} -DIFF=${DIFF:-diff} -JIRACLI=${JIRA:-jira} -FINDBUGS_HOME=${FINDBUGS_HOME} -FORREST_HOME=${FORREST_HOME} -ECLIPSE_HOME=${ECLIPSE_HOME} +## @description Setup the default global variables +## @audience public +## @stability stable +## @replaceable no +function setup_defaults +{ + if [[ -z "${MAVEN_HOME:-}" ]]; then + MVN=mvn + else + MVN=${MAVEN_HOME}/bin/mvn + fi -############################################################################### -printUsage() { - echo "Usage: $0 [options] patch-file | defect-number" + PROJECT_NAME=hadoop + JENKINS=false + PATCH_DIR=/tmp/${PROJECT_NAME}-test-patch/$$ + BASEDIR=$(pwd) + + FINDBUGS_HOME=${FINDBUGS_HOME:-} + ECLIPSE_HOME=${ECLIPSE_HOME:-} + BUILD_NATIVE=${BUILD_NATIVE:-true} + PATCH_BRANCH="" + CHANGED_MODULES="" + USER_MODULE_LIST="" + OFFLINE=false + CHANGED_FILES="" + REEXECED=false + RESETREPO=false + ISSUE="" + ISSUE_RE='^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$' + TIMER=$(date +"%s") + + OSTYPE=$(uname -s) + + # Solaris needs POSIX, not SVID + case ${OSTYPE} in + SunOS) + PS=${PS:-ps} + AWK=${AWK:-/usr/xpg4/bin/awk} + SED=${SED:-/usr/xpg4/bin/sed} + WGET=${WGET:-wget} + GIT=${GIT:-git} + EGREP=${EGREP:-/usr/xpg4/bin/egrep} + GREP=${GREP:-/usr/xpg4/bin/grep} + PATCH=${PATCH:-patch} + DIFF=${DIFF:-diff} + JIRACLI=${JIRA:-jira} + ;; + *) + PS=${PS:-ps} + AWK=${AWK:-awk} + SED=${SED:-sed} + WGET=${WGET:-wget} + GIT=${GIT:-git} + EGREP=${EGREP:-egrep} + GREP=${GREP:-grep} + PATCH=${PATCH:-patch} + DIFF=${DIFF:-diff} + JIRACLI=${JIRA:-jira} + ;; + esac + + declare -a JIRA_COMMENT_TABLE + declare -a JIRA_FOOTER_TABLE + declare -a JIRA_HEADER + declare -a JIRA_TEST_TABLE + + JFC=0 + JTC=0 + JTT=0 + RESULT=0 +} + +## @description Print a message to stderr +## @audience public +## @stability stable +## @replaceable no +## @param string +function hadoop_error +{ + echo "$*" 1>&2 +} + +## @description Print a message to stderr if --debug is turned on +## @audience public +## @stability stable +## @replaceable no +## @param string +function hadoop_debug +{ + if [[ -n "${HADOOP_SHELL_SCRIPT_DEBUG}" ]]; then + echo "[$(date) DEBUG]: $*" 1>&2 + fi +} + +## @description Activate the local timer +## @audience public +## @stability stable +## @replaceable no +function start_clock +{ + hadoop_debug "Start clock" + TIMER=$(date +"%s") +} + +## @description Print the elapsed time in seconds since the start of the local timer +## @audience public +## @stability stable +## @replaceable no +function stop_clock +{ + local -r stoptime=$(date +"%s") + local -r elapsed=$((stoptime-TIMER)) + hadoop_debug "Stop clock" + + echo ${elapsed} +} + +## @description Print the elapsed time in seconds since the start of the global timer +## @audience private +## @stability stable +## @replaceable no +function stop_global_clock +{ + local -r stoptime=$(date +"%s") + local -r elapsed=$((stoptime-GLOBALTIMER)) + hadoop_debug "Stop global clock" + + echo ${elapsed} +} + +## @description Add time to the local timer +## @audience public +## @stability stable +## @replaceable no +## @param seconds +function offset_clock +{ + ((TIMER=TIMER-$1)) +} + +## @description Add to the header of the display +## @audience public +## @stability stable +## @replaceable no +## @param string +function add_jira_header +{ + JIRA_HEADER[${JHC}]="| $* |" + JHC=$(( JHC+1 )) +} + +## @description Add to the output table. If the first parameter is a number +## @description that is the vote for that column and calculates the elapsed time +## @description based upon the last start_clock(). If it the string null, then it is +## @description a special entry that signifies extra +## @description content for the final column. The second parameter is the reporting +## @description subsystem (or test) that is providing the vote. The second parameter +## @description is always required. The third parameter is any extra verbage that goes +## @description with that subsystem. +## @audience public +## @stability stable +## @replaceable no +## @param +1/0/-1/null +## @param subsystem +## @param string +## @return Elapsed time display +function add_jira_table +{ + local value=$1 + local subsystem=$2 + shift 2 + + local color + local calctime=0 + + local -r elapsed=$(stop_clock) + + if [[ ${elapsed} -lt 0 ]]; then + calctime="N/A" + else + printf -v calctime "%3sm %02ss" $((elapsed/60)) $((elapsed%60)) + fi + + echo "" + echo "Elapsed time: ${calctime}" + echo "" + + case ${value} in + 1|+1) + value="+1" + color="green" + ;; + -1) + color="red" + ;; + 0) + color="blue" + ;; + null) + ;; + esac + + if [[ -z ${color} ]]; then + JIRA_COMMENT_TABLE[${JTC}]="| | ${subsystem} | | ${*:-} |" + JTC=$(( JTC+1 )) + else + JIRA_COMMENT_TABLE[${JTC}]="| {color:${color}}${value}{color} | ${subsystem} | ${calctime} | $* |" + JTC=$(( JTC+1 )) + fi +} + +## @description Put the final elapsed time at the bottom of the table. +## @audience private +## @stability stable +## @replaceable no +function close_jira_table +{ + + local -r elapsed=$(stop_global_clock) + + if [[ ${elapsed} -lt 0 ]]; then + calctime="N/A" + else + printf -v calctime "%3sm %02ss" $((elapsed/60)) $((elapsed%60)) + fi + + echo "" + echo "Total Elapsed time: ${calctime}" + echo "" + + + JIRA_COMMENT_TABLE[${JTC}]="| | | ${calctime} | |" + JTC=$(( JTC+1 )) +} + +## @description Add to the footer of the display. @@BASE@@ will get replaced with the +## @description correct location for the local filesystem in dev mode or the URL for +## @description Jenkins mode. +## @audience public +## @stability stable +## @replaceable no +## @param subsystem +## @param string +function add_jira_footer +{ + local subsystem=$1 + shift 1 + + JIRA_FOOTER_TABLE[${JFC}]="| ${subsystem} | $* |" + JFC=$(( JFC+1 )) +} + +## @description Special table just for unit test failures +## @audience public +## @stability stable +## @replaceable no +## @param failurereason +## @param testlist +function add_jira_test_table +{ + local failure=$1 + shift 1 + + JIRA_TEST_TABLE[${JTT}]="| ${failure} | $* |" + JTT=$(( JTT+1 )) +} + +## @description Large display for the user console +## @audience public +## @stability stable +## @replaceable no +## @param string +## @return large chunk of text +function big_console_header +{ + local text="$*" + local spacing=$(( (75+${#text}) /2 )) + printf "\n\n" + echo "============================================================================" + echo "============================================================================" + printf "%*s\n" ${spacing} "${text}" + echo "============================================================================" + echo "============================================================================" + printf "\n\n" +} + +## @description Remove {color} tags from a string +## @audience public +## @stability stable +## @replaceable no +## @param string +## @return string +function colorstripper +{ + local string=$1 + shift 1 + + local green="" + local white="" + local red="" + local blue="" + + echo "${string}" | \ + ${SED} -e "s,{color:red},${red},g" \ + -e "s,{color:green},${green},g" \ + -e "s,{color:blue},${blue},g" \ + -e "s,{color},${white},g" +} + +## @description Find the largest size of a column of an array +## @audience private +## @stability evolving +## @replaceable no +## @return size +function findlargest +{ + local column=$1 + shift + local a=("$@") + local sizeofa=${#a[@]} + local i=0 + + until [[ ${i} -gt ${sizeofa} ]]; do + # shellcheck disable=SC2086 + string=$( echo ${a[$i]} | cut -f$((column + 1)) -d\| ) + if [[ ${#string} -gt $maxlen ]]; then + maxlen=${#string} + fi + i=$((i+1)) + done + echo "${maxlen}" +} + +## @description Verify that ${JAVA_HOME} is defined +## @audience public +## @stability stable +## @replaceable no +## @return 1 - no JAVA_HOME +## @return 0 - JAVA_HOME defined +function find_java_home +{ + start_clock + if [[ -z ${JAVA_HOME:-} ]]; then + case $(uname -s) in + Darwin) + if [[ -z "${JAVA_HOME}" ]]; then + if [[ -x /usr/libexec/java_home ]]; then + JAVA_HOME="$(/usr/libexec/java_home)" + export JAVA_HOME + else + export JAVA_HOME=/Library/Java/Home + fi + fi + ;; + *) + ;; + esac + fi + + if [[ -z ${JAVA_HOME:-} ]]; then + echo "JAVA_HOME is not defined." + add_jira_table -1 pre-patch "JAVA_HOME is not defined." + return 1 + fi + return 0 +} + +## @description Print the command to be executing to the screen. Then +## @description run the command, sending stdout and stderr to the given filename +## @description This will also ensure that any directories in ${BASEDIR} have +## @description the exec bit set as a pre-exec step. +## @audience public +## @stability stable +## @param filename +## @param command +## @param [..] +## @replaceable no +## @returns $? +function echo_and_redirect +{ + logfile=$1 + shift + find "${BASEDIR}" -type d -exec chmod +x {} \; + echo "${*} > ${logfile} 2>&1" + "${@}" > "${logfile}" 2>&1 +} + +## @description Print the usage information +## @audience public +## @stability stable +## @replaceable no +function hadoop_usage +{ + local -r up=$(echo ${PROJECT_NAME} | tr '[:lower:]' '[:upper:]') + + echo "Usage: test-patch.sh [options] patch-file | issue-number | http" echo echo "Where:" echo " patch-file is a local patch file containing the changes to test" - echo " defect-number is a JIRA defect number (e.g. 'HADOOP-1234') to test (Jenkins only)" + echo " issue-number is a 'Patch Available' JIRA defect number (e.g. '${up}-9902') to test" + echo " http is an HTTP address to download the patch file" echo echo "Options:" - echo "--patch-dir= The directory for working and output files (default '/tmp')" echo "--basedir= The directory to apply the patch to (default current directory)" - echo "--mvn-cmd= The 'mvn' command to use (default \$MAVEN_HOME/bin/mvn, or 'mvn')" - echo "--ps-cmd= The 'ps' command to use (default 'ps')" + echo "--build-native= If true, then build native components (default 'true')" + echo "--debug If set, then output some extra stuff to stderr" + echo "--dirty-workspace Allow the local git workspace to have uncommitted changes" + echo "--findbugs-home= Findbugs home directory (default FINDBUGS_HOME environment variable)" + echo "--modulelist= Specify additional modules to test (comma delimited)" + echo "--offline Avoid connecting to the Internet" + echo "--patch-dir= The directory for working and output files (default '/tmp/${PROJECT_NAME}-test-patch/pid')" + echo "--resetrepo Forcibly clean the repo" + echo "--run-tests Run all relevant tests below the base directory" + echo "--testlist= Specify which subsystem tests to use (comma delimited)" + + echo "Shell binary overrides:" echo "--awk-cmd= The 'awk' command to use (default 'awk')" + echo "--diff-cmd= The 'diff' command to use (default 'diff')" echo "--git-cmd= The 'git' command to use (default 'git')" echo "--grep-cmd= The 'grep' command to use (default 'grep')" + echo "--mvn-cmd= The 'mvn' command to use (default \${MAVEN_HOME}/bin/mvn, or 'mvn')" echo "--patch-cmd= The 'patch' command to use (default 'patch')" - echo "--diff-cmd= The 'diff' command to use (default 'diff')" - echo "--findbugs-home= Findbugs home directory (default FINDBUGS_HOME environment variable)" - echo "--forrest-home= Forrest home directory (default FORREST_HOME environment variable)" - echo "--dirty-workspace Allow the local git workspace to have uncommitted changes" - echo "--run-tests Run all tests below the base directory" - echo "--build-native= If true, then build native components (default 'true')" + echo "--ps-cmd= The 'ps' command to use (default 'ps')" + echo "--sed-cmd= The 'sed' command to use (default 'sed')" + echo echo "Jenkins-only options:" echo "--jenkins Run by Jenkins (runs tests and posts results to JIRA)" - echo "--support-dir= The directory to find support files in" - echo "--wget-cmd= The 'wget' command to use (default 'wget')" + echo "--eclipse-home= Eclipse home directory (default ECLIPSE_HOME environment variable)" echo "--jira-cmd= The 'jira' command to use (default 'jira')" echo "--jira-password= The password for the 'jira' command" - echo "--eclipse-home= Eclipse home directory (default ECLIPSE_HOME environment variable)" + echo "--wget-cmd= The 'wget' command to use (default 'wget')" } -############################################################################### -parseArgs() { - for i in $* - do - case $i in - --jenkins) - JENKINS=true +## @description Interpret the command line parameters +## @audience private +## @stability stable +## @replaceable no +## @params $@ +## @return May exit on failure +function parse_args +{ + local i + local j + + for i in "$@"; do + case ${i} in + --awk-cmd=*) + AWK=${i#*=} ;; - --patch-dir=*) - PATCH_DIR=${i#*=} + --basedir=*) + BASEDIR=${i#*=} ;; - --support-dir=*) - SUPPORT_DIR=${i#*=} + --build-native=*) + BUILD_NATIVE=${i#*=} ;; - --basedir=*) - BASEDIR=${i#*=} + --debug) + HADOOP_SHELL_SCRIPT_DEBUG=true ;; - --mvn-cmd=*) - MVN=${i#*=} + --diff-cmd=*) + DIFF=${i#*=} ;; - --ps-cmd=*) - PS=${i#*=} + --dirty-workspace) + DIRTY_WORKSPACE=true ;; - --awk-cmd=*) - AWK=${i#*=} + --eclipse-home=*) + ECLIPSE_HOME=${i#*=} ;; - --wget-cmd=*) - WGET=${i#*=} + --findbugs-home=*) + FINDBUGS_HOME=${i#*=} ;; - --git-cmd=*) - GIT=${i#*=} + --git-cmd=*) + GIT=${i#*=} ;; - --grep-cmd=*) - GREP=${i#*=} + --grep-cmd=*) + GREP=${i#*=} ;; - --patch-cmd=*) - PATCH=${i#*=} + --java-home) + JAVA_HOME=${i#*=} ;; - --diff-cmd=*) - DIFF=${i#*=} + --jenkins) + JENKINS=true ;; - --jira-cmd=*) - JIRACLI=${i#*=} + --jira-cmd=*) + JIRACLI=${i#*=} ;; - --jira-password=*) - JIRA_PASSWD=${i#*=} + --jira-password=*) + JIRA_PASSWD=${i#*=} ;; - --findbugs-home=*) - FINDBUGS_HOME=${i#*=} + --modulelist=*) + USER_MODULE_LIST=${i#*=} + USER_MODULE_LIST=${USER_MODULE_LIST//,/ } + hadoop_debug "Manually forcing modules ${USER_MODULE_LIST}" ;; - --forrest-home=*) - FORREST_HOME=${i#*=} + --mvn-cmd=*) + MVN=${i#*=} ;; - --eclipse-home=*) - ECLIPSE_HOME=${i#*=} + --offline) + OFFLINE=true ;; - --dirty-workspace) - DIRTY_WORKSPACE=true + --patch-cmd=*) + PATCH=${i#*=} ;; - --run-tests) - RUN_TESTS=true + --patch-dir=*) + PATCH_DIR=${i#*=} ;; - --build-native=*) - BUILD_NATIVE=${i#*=} + --ps-cmd=*) + PS=${i#*=} ;; - *) - PATCH_OR_DEFECT=$i + --reexec) + REEXECED=true + start_clock + add_jira_table 0 reexec "dev-support patch detected." + ;; + --resetrepo) + RESETREPO=true + ;; + --run-tests) + RUN_TESTS=true + ;; + --testlist=*) + testlist=${i#*=} + testlist=${testlist//,/ } + for j in ${testlist}; do + hadoop_debug "Manually adding patch test subsystem ${j}" + add_test "${j}" + done + ;; + --wget-cmd=*) + WGET=${i#*=} + ;; + *) + PATCH_OR_ISSUE=${i} ;; esac done - if [[ $BUILD_NATIVE == "true" ]] ; then + + # if we get a relative path, turn it absolute + BASEDIR=$(cd -P -- "${BASEDIR}" >/dev/null && pwd -P) + + if [[ ${BUILD_NATIVE} == "true" ]] ; then NATIVE_PROFILE=-Pnative REQUIRE_TEST_LIB_HADOOP=-Drequire.test.libhadoop fi - if [ -z "$PATCH_OR_DEFECT" ]; then - printUsage + if [[ -z "${PATCH_OR_ISSUE}" ]]; then + hadoop_usage exit 1 fi - if [[ $JENKINS == "true" ]] ; then + if [[ ${JENKINS} == "true" ]] ; then echo "Running in Jenkins mode" - defect=$PATCH_OR_DEFECT - ECLIPSE_PROPERTY="-Declipse.home=$ECLIPSE_HOME" + ISSUE=${PATCH_OR_ISSUE} + RESETREPO=true + # shellcheck disable=SC2034 + ECLIPSE_PROPERTY="-Declipse.home=${ECLIPSE_HOME}" else - echo "Running in developer mode" - JENKINS=false - ### PATCH_FILE contains the location of the patchfile - PATCH_FILE=$PATCH_OR_DEFECT - if [[ ! -e "$PATCH_FILE" ]] ; then - echo "Unable to locate the patch file $PATCH_FILE" - cleanupAndExit 0 + if [[ ${RESETREPO} == "true" ]] ; then + echo "Running in destructive (--resetrepo) developer mode" + else + echo "Running in developer mode" fi - ### Check if $PATCH_DIR exists. If it does not exist, create a new directory - if [[ ! -e "$PATCH_DIR" ]] ; then - mkdir "$PATCH_DIR" - if [[ $? == 0 ]] ; then - echo "$PATCH_DIR has been created" - else - echo "Unable to create $PATCH_DIR" - cleanupAndExit 0 + JENKINS=false + fi + + if [[ ! -d ${PATCH_DIR} ]]; then + mkdir -p "${PATCH_DIR}" + if [[ $? == 0 ]] ; then + echo "${PATCH_DIR} has been created" + else + echo "Unable to create ${PATCH_DIR}" + cleanup_and_exit 1 + fi + fi +} + +## @description Locate the pom.xml file for a given directory +## @audience private +## @stability stable +## @replaceable no +## @return directory containing the pom.xml +function find_pom_dir +{ + local dir + + dir=$(dirname "$1") + + hadoop_debug "Find pom dir for: ${dir}" + + while builtin true; do + if [[ -f "${dir}/pom.xml" ]];then + echo "${dir}" + hadoop_debug "Found: ${dir}" + return + else + dir=$(dirname "${dir}") + fi + done +} + +## @description List of files that ${PATCH_DIR}/patch modifies +## @audience private +## @stability stable +## @replaceable no +## @return None; sets ${CHANGED_FILES} +function find_changed_files +{ + # get a list of all of the files that have been changed, + # except for /dev/null (which would be present for new files). + # Additionally, remove any a/ b/ patterns at the front + # of the patch filenames + CHANGED_FILES=$(${GREP} -E '^(\+\+\+|---) ' "${PATCH_DIR}/patch" \ + | ${SED} \ + -e 's,^....,,' \ + -e 's,^[ab]/,,' \ + | ${GREP} -v /dev/null \ + | sort -u) +} + +## @description Find the modules of the maven build that ${PATCH_DIR}/patch modifies +## @audience private +## @stability stable +## @replaceable no +## @return None; sets ${CHANGED_MODULES} +function find_changed_modules +{ + # Come up with a list of changed files into ${TMP} + local pomdirs + local module + local pommods + + # Now find all the modules that were changed + for file in ${CHANGED_FILES}; do + #shellcheck disable=SC2086 + pomdirs="${pomdirs} $(find_pom_dir ${file})" + done + + # Filter out modules without code + for module in ${pomdirs}; do + ${GREP} "pom" "${module}/pom.xml" > /dev/null + if [[ "$?" != 0 ]]; then + pommods="${pommods} ${module}" + fi + done + + #shellcheck disable=SC2086 + CHANGED_MODULES=$(echo ${pommods} ${USER_MODULE_LIST} | tr ' ' '\n' | sort -u) +} + +## @description git checkout the appropriate branch to test. Additionally, this calls +## @description 'determine_issue' and 'determine_branch' based upon the context provided +## @description in ${PATCH_DIR} and in git after checkout. +## @audience private +## @stability stable +## @replaceable no +## @return 0 on success. May exit on failure. +function git_checkout +{ + local currentbranch + + big_console_header "Confirming git environment" + + if [[ ${RESETREPO} == "true" ]] ; then + cd "${BASEDIR}" + ${GIT} reset --hard + if [[ $? != 0 ]]; then + hadoop_error "ERROR: git reset is failing" + cleanup_and_exit 1 + fi + ${GIT} clean -xdf + if [[ $? != 0 ]]; then + hadoop_error "ERROR: git clean is failing" + cleanup_and_exit 1 + fi + + ${GIT} checkout --force trunk + if [[ $? != 0 ]]; then + hadoop_error "ERROR: git checkout --force trunk is failing" + cleanup_and_exit 1 + fi + + determine_branch + if [[ ${PATCH_BRANCH} =~ ^git ]]; then + PATCH_BRANCH=$(echo "${PATCH_BRANCH}" | cut -dt -f2) + fi + + # we need to explicitly fetch in case the + # git ref hasn't been brought in tree yet + if [[ ${OFFLINE} == false ]]; then + ${GIT} pull --rebase + if [[ $? != 0 ]]; then + hadoop_error "ERROR: git pull is failing" + cleanup_and_exit 1 fi fi - ### Obtain the patch filename to append it to the version number - defect=`basename $PATCH_FILE` - fi -} - -############################################################################### -checkout () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Testing patch for ${defect}." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - ### When run by a developer, if the workspace contains modifications, do not continue - ### unless the --dirty-workspace option was set - status=`$GIT status --porcelain` - if [[ $JENKINS == "false" ]] ; then - if [[ "$status" != "" && -z $DIRTY_WORKSPACE ]] ; then - echo "ERROR: can't run in a workspace that contains the following modifications" - echo "$status" - cleanupAndExit 1 + # forcibly checkout this branch or git ref + ${GIT} checkout --force "${PATCH_BRANCH}" + if [[ $? != 0 ]]; then + hadoop_error "ERROR: git checkout ${PATCH_BRANCH} is failing" + cleanup_and_exit 1 fi - echo - else - cd $BASEDIR - $GIT reset --hard - $GIT clean -xdf - $GIT checkout trunk - $GIT pull --rebase - fi - GIT_REVISION=`git rev-parse --verify --short HEAD` - return $? -} -############################################################################### -downloadPatch () { - ### Download latest patch file (ignoring .htm and .html) when run from patch process - if [[ $JENKINS == "true" ]] ; then - $WGET -q -O $PATCH_DIR/jira http://issues.apache.org/jira/browse/$defect - if [[ `$GREP -c 'Patch Available' $PATCH_DIR/jira` == 0 ]] ; then - echo "$defect is not \"Patch Available\". Exiting." - cleanupAndExit 0 - fi - relativePatchURL=`$GREP -o '"/jira/secure/attachment/[0-9]*/[^"]*' $PATCH_DIR/jira | $GREP -v -e 'htm[l]*$' | sort | tail -1 | $GREP -o '/jira/secure/attachment/[0-9]*/[^"]*'` - patchURL="http://issues.apache.org${relativePatchURL}" - patchNum=`echo $patchURL | $GREP -o '[0-9]*/' | $GREP -o '[0-9]*'` - echo "$defect patch is being downloaded at `date` from" - echo "$patchURL" - $WGET -q -O $PATCH_DIR/patch $patchURL - VERSION=${GIT_REVISION}_${defect}_PATCH-${patchNum} - JIRA_COMMENT="Here are the results of testing the latest attachment - $patchURL - against trunk revision ${GIT_REVISION}." - - ### Copy in any supporting files needed by this process - cp -r $SUPPORT_DIR/lib/* ./lib - #PENDING: cp -f $SUPPORT_DIR/etc/checkstyle* ./src/test - ### Copy the patch file to $PATCH_DIR else - VERSION=PATCH-${defect} - cp $PATCH_FILE $PATCH_DIR/patch - if [[ $? == 0 ]] ; then - echo "Patch file $PATCH_FILE copied to $PATCH_DIR" + cd "${BASEDIR}" + if [[ ! -d .git ]]; then + hadoop_error "ERROR: ${BASEDIR} is not a git repo." + cleanup_and_exit 1 + fi + + status=$(${GIT} status --porcelain) + if [[ "${status}" != "" && -z ${DIRTY_WORKSPACE} ]] ; then + hadoop_error "ERROR: --dirty-workspace option not provided." + hadoop_error "ERROR: can't run in a workspace that contains the following modifications" + hadoop_error "${status}" + cleanup_and_exit 1 + fi + + determine_branch + if [[ ${PATCH_BRANCH} =~ ^git ]]; then + PATCH_BRANCH=$(echo "${PATCH_BRANCH}" | cut -dt -f2) + fi + + currentbranch=$(${GIT} rev-parse --abbrev-ref HEAD) + if [[ "${currentbranch}" != "${PATCH_BRANCH}" ]];then + echo "WARNING: Current git branch is ${currentbranch} but patch is built for ${PATCH_BRANCH}." + echo "WARNING: Continuing anyway..." + PATCH_BRANCH=${currentbranch} + fi + fi + + determine_issue + + GIT_REVISION=$(${GIT} rev-parse --verify --short HEAD) + # shellcheck disable=SC2034 + VERSION=${GIT_REVISION}_${ISSUE}_PATCH-${patchNum} + + if [[ "${ISSUE}" == 'Unknown' ]]; then + echo "Testing patch on ${PATCH_BRANCH}." + else + echo "Testing ${ISSUE} patch on ${PATCH_BRANCH}." + fi + + add_jira_footer "git revision" "${PATCH_BRANCH} / ${GIT_REVISION}" + return 0 +} + +## @description Confirm the source environment is compilable +## @audience private +## @stability stable +## @replaceable no +## @return 0 on success +## @return 1 on failure +function precheck_without_patch +{ + local -r mypwd=$(pwd) + + big_console_header "Pre-patch ${PATCH_BRANCH} Java verification" + + start_clock + + verify_needed_test javac + + if [[ $? == 1 ]]; then + echo "Compiling ${mypwd}" + echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}JavacWarnings.txt" "${MVN}" clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch + if [[ $? != 0 ]] ; then + echo "${PATCH_BRANCH} compilation is broken?" + add_jira_table -1 pre-patch "${PATCH_BRANCH} compilation may be broken." + return 1 + fi + else + echo "Patch does not appear to need javac tests." + fi + + verify_needed_test javadoc + + if [[ $? == 1 ]]; then + echo "Javadoc'ing ${mypwd}" + echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarnings.txt" "${MVN}" clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess + if [[ $? != 0 ]] ; then + echo "Pre-patch ${PATCH_BRANCH} javadoc compilation is broken?" + add_jira_table -1 pre-patch "Pre-patch ${PATCH_BRANCH} JavaDoc compilation may be broken." + return 1 + fi + else + echo "Patch does not appear to need javadoc tests." + fi + + verify_needed_test site + + if [[ $? == 1 ]]; then + echo "site creation for ${mypwd}" + echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}SiteWarnings.txt" "${MVN}" clean site site:stage -DskipTests -Dmaven.javadoc.skip=true -D${PROJECT_NAME}PatchProcess + if [[ $? != 0 ]] ; then + echo "Pre-patch ${PATCH_BRANCH} site compilation is broken?" + add_jira_table -1 pre-patch "Pre-patch ${PATCH_BRANCH} site compilation may be broken." + return 1 + fi + else + echo "Patch does not appear to need site tests." + fi + + add_jira_table 0 pre-patch "Pre-patch ${PATCH_BRANCH} compilation is healthy." + return 0 +} + +## @description Confirm the given branch is a member of the list of space +## @description delimited branches or a git ref +## @audience private +## @stability evolving +## @replaceable no +## @param branch +## @param branchlist +## @return 0 on success +## @return 1 on failure +function verify_valid_branch +{ + local branches=$1 + local check=$2 + local i + + if [[ ${check} =~ ^git ]]; then + ref=$(echo "${check}" | cut -f2 -dt) + count=$(echo "${ref}" | wc -c | tr -d ' ') + + if [[ ${count} == 8 || ${count} == 41 ]]; then + return 0 + fi + return 1 + fi + + for i in ${branches}; do + if [[ "${i}" == "${check}" ]]; then + return 0 + fi + done + return 1 +} + +## @description Try to guess the branch being tested using a variety of heuristics +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success, with PATCH_BRANCH updated appropriately +## @return 1 on failure, with PATCH_BRANCH updated to "trunk" +function determine_branch +{ + local allbranches + local patchnamechunk + + hadoop_debug "Determine branch" + + # something has already set this, so move on + if [[ -n ${PATCH_BRANCH} ]]; then + return + fi + + pushd "${BASEDIR}" > /dev/null + + # developer mode, existing checkout, whatever + if [[ "${DIRTY_WORKSPACE}" == true ]];then + PATCH_BRANCH=$(${GIT} rev-parse --abbrev-ref HEAD) + echo "dirty workspace mode; applying against existing branch" + return + fi + + allbranches=$(${GIT} branch -r | tr -d ' ' | ${SED} -e s,origin/,,g) + + # shellcheck disable=SC2016 + patchnamechunk=$(echo "${PATCH_OR_ISSUE}" | ${AWK} -F/ '{print $NF}') + + # ISSUE.branch.##.patch + PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f2 -d. ) + verify_valid_branch "${allbranches}" "${PATCH_BRANCH}" + if [[ $? == 0 ]]; then + return + fi + + # ISSUE-branch-##.patch + PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | cut -f1,2 -d-) + verify_valid_branch "${allbranches}" "${PATCH_BRANCH}" + if [[ $? == 0 ]]; then + return + fi + + # ISSUE-##.patch.branch + # shellcheck disable=SC2016 + PATCH_BRANCH=$(echo "${patchnamechunk}" | ${AWK} -F. '{print $NF}') + verify_valid_branch "${allbranches}" "${PATCH_BRANCH}" + if [[ $? == 0 ]]; then + return + fi + + # ISSUE-branch.##.patch + # shellcheck disable=SC2016 + PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | ${AWK} -F. '{print $(NF-2)}' 2>/dev/null) + verify_valid_branch "${allbranches}" "${PATCH_BRANCH}" + if [[ $? == 0 ]]; then + return + fi + + PATCH_BRANCH=trunk + + popd >/dev/null +} + +## @description Try to guess the issue being tested using a variety of heuristics +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success, with ISSUE updated appropriately +## @return 1 on failure, with ISSUE updated to "Unknown" +function determine_issue +{ + local patchnamechunk + local maybeissue + + hadoop_debug "Determine issue" + + # we can shortcut jenkins + if [[ ${JENKINS} == true ]]; then + ISSUE=${PATCH_OR_ISSUE} + fi + + # shellcheck disable=SC2016 + patchnamechunk=$(echo "${PATCH_OR_ISSUE}" | ${AWK} -F/ '{print $NF}') + + maybeissue=$(echo "${patchnamechunk}" | cut -f1,2 -d-) + + if [[ ${maybeissue} =~ ${ISSUE_RE} ]]; then + ISSUE=${maybeissue} + return 0 + fi + + ISSUE="Unknown" + return 1 +} + +## @description Add the given test type +## @audience public +## @stability stable +## @replaceable yes +## @param test +function add_test +{ + local testname=$1 + + hadoop_debug "Testing against ${testname}" + + if [[ -z ${NEEDED_TESTS} ]]; then + hadoop_debug "Setting tests to ${testname}" + NEEDED_TESTS=${testname} + elif [[ ! ${NEEDED_TESTS} =~ ${testname} ]] ; then + hadoop_debug "Adding ${testname}" + NEEDED_TESTS="${NEEDED_TESTS} ${testname}" + fi +} + +## @description Verify if a given test was requested +## @audience public +## @stability stable +## @replaceable yes +## @param test +## @return 1 = yes +## @return 0 = no +function verify_needed_test +{ + local i=$1 + + if [[ ${NEEDED_TESTS} =~ $i ]]; then + return 1 + fi + return 0 +} + +## @description Use some heuristics to determine which long running +## @description tests to run +## @audience private +## @stability stable +## @replaceable no +function determine_needed_tests +{ + local i + + for i in ${CHANGED_FILES}; do + if [[ ${i} =~ src/main/webapp ]]; then + hadoop_debug "tests/webapp: ${i}" + elif [[ ${i} =~ \.sh + || ${i} =~ \.cmd + ]]; then + hadoop_debug "tests/shell: ${i}" + elif [[ ${i} =~ \.md$ + || ${i} =~ \.md\.vm$ + || ${i} =~ src/site + || ${i} =~ src/main/docs + ]]; then + hadoop_debug "tests/site: ${i}" + add_test site + elif [[ ${i} =~ \.c$ + || ${i} =~ \.cc$ + || ${i} =~ \.h$ + || ${i} =~ \.hh$ + || ${i} =~ \.proto$ + || ${i} =~ src/test + || ${i} =~ \.cmake$ + || ${i} =~ CMakeLists.txt + ]]; then + hadoop_debug "tests/units: ${i}" + add_test javac + add_test unit + elif [[ ${i} =~ pom.xml$ + || ${i} =~ \.java$ + || ${i} =~ src/main + ]]; then + hadoop_debug "tests/javadoc+units: ${i}" + add_test javadoc + add_test javac + add_test unit + fi + + if [[ ${i} =~ \.java$ ]]; then + add_test findbugs + fi + + for plugin in ${PLUGINS}; do + if declare -f ${plugin}_filefilter >/dev/null 2>&1; then + "${plugin}_filefilter" "${i}" + fi + done + done + + add_jira_footer "Optional Tests" "${NEEDED_TESTS}" +} + +## @description Given ${PATCH_ISSUE}, determine what type of patch file is in use, and do the +## @description necessary work to place it into ${PATCH_DIR}/patch. +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure, may exit +function locate_patch +{ + hadoop_debug "locate patch" + + if [[ -f ${PATCH_OR_ISSUE} ]]; then + PATCH_FILE="${PATCH_OR_ISSUE}" + else + if [[ ${PATCH_OR_ISSUE} =~ ^http ]]; then + echo "Patch is being downloaded at $(date) from" + patchURL="${PATCH_OR_ISSUE}" else - echo "Could not copy $PATCH_FILE to $PATCH_DIR" - cleanupAndExit 0 + ${WGET} -q -O "${PATCH_DIR}/jira" "http://issues.apache.org/jira/browse/${PATCH_OR_ISSUE}" + + if [[ $? != 0 ]];then + hadoop_error "ERROR: Unable to determine what ${PATCH_OR_ISSUE} may reference." + cleanup_and_exit 1 + fi + + if [[ $(${GREP} -c 'Patch Available' "${PATCH_DIR}/jira") == 0 ]] ; then + if [[ ${JENKINS} == true ]]; then + hadoop_error "ERROR: ${PATCH_OR_ISSUE} is not \"Patch Available\"." + cleanup_and_exit 1 + else + hadoop_error "WARNING: ${PATCH_OR_ISSUE} is not \"Patch Available\"." + fi + fi + + relativePatchURL=$(${GREP} -o '"/jira/secure/attachment/[0-9]*/[^"]*' "${PATCH_DIR}/jira" | ${GREP} -v -e 'htm[l]*$' | sort | tail -1 | ${GREP} -o '/jira/secure/attachment/[0-9]*/[^"]*') + patchURL="http://issues.apache.org${relativePatchURL}" + if [[ ! ${patchURL} =~ \.patch$ ]]; then + hadoop_error "ERROR: ${patchURL} is not a patch file." + cleanup_and_exit 1 + fi + patchNum=$(echo "${patchURL}" | ${GREP} -o '[0-9]*/' | ${GREP} -o '[0-9]*') + echo "${ISSUE} patch is being downloaded at $(date) from" + fi + echo "${patchURL}" + add_jira_footer "Patch URL" "${patchURL}" + ${WGET} -q -O "${PATCH_DIR}/patch" "${patchURL}" + if [[ $? != 0 ]];then + hadoop_error "ERROR: ${PATCH_OR_ISSUE} could not be downloaded." + cleanup_and_exit 1 + fi + PATCH_FILE="${PATCH_DIR}/patch" + fi + + if [[ ! -f "${PATCH_DIR}/patch" ]]; then + cp "${PATCH_FILE}" "${PATCH_DIR}/patch" + if [[ $? == 0 ]] ; then + echo "Patch file ${PATCH_FILE} copied to ${PATCH_DIR}" + else + hadoop_error "ERROR: Could not copy ${PATCH_FILE} to ${PATCH_DIR}" + cleanup_and_exit 1 fi fi } -############################################################################### -verifyPatch () { +## @description Given ${PATCH_DIR}/patch, verify the patch is good using ${BINDIR}/smart-apply-patch.sh +## @description in dryrun mode. +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function verify_patch_file +{ # Before building, check to make sure that the patch is valid - $bindir/smart-apply-patch.sh $PATCH_DIR/patch dryrun + export PATCH + + "${BINDIR}/smart-apply-patch.sh" "${PATCH_DIR}/patch" dryrun if [[ $? != 0 ]] ; then echo "PATCH APPLICATION FAILED" - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 patch{color}. The patch command could not apply the patch." + add_jira_table -1 patch "The patch command could not apply the patch during dryrun." return 1 else return 0 fi } -############################################################################### -prebuildWithoutPatch () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Pre-build trunk to verify trunk stability and javac warnings" - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - if [[ ! -d hadoop-common-project ]]; then - cd $bindir/.. - echo "Compiling $(pwd)" - echo "$MVN clean test -DskipTests > $PATCH_DIR/trunkCompile.txt 2>&1" - $MVN clean test -DskipTests > $PATCH_DIR/trunkCompile.txt 2>&1 - if [[ $? != 0 ]] ; then - echo "Top-level trunk compilation is broken?" - JIRA_COMMENT="$JIRA_COMMENT +## @description Given ${PATCH_DIR}/patch, apply the patch using ${BINDIR}/smart-apply-patch.sh +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return exit on failure +function apply_patch_file +{ + big_console_header "Applying patch" - {color:red}-1 patch{color}. Top-level trunk compilation may be broken." - return 1 - fi - cd - - fi - echo "Compiling $(pwd)" - if [[ -d "$(pwd)"/hadoop-hdfs-project/hadoop-hdfs/target/test/data/dfs ]]; then - echo "Changing permission $(pwd)/hadoop-hdfs-project/hadoop-hdfs/target/test/data/dfs to avoid broken builds " - chmod +x -R "$(pwd)/hadoop-hdfs-project/hadoop-hdfs/target/test/data/dfs" - fi - echo "$MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch > $PATCH_DIR/trunkJavacWarnings.txt 2>&1" - $MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch > $PATCH_DIR/trunkJavacWarnings.txt 2>&1 - if [[ $? != 0 ]] ; then - echo "Trunk compilation is broken?" - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 patch{color}. Trunk compilation may be broken." - return 1 - fi - - echo "$MVN clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/trunkJavadocWarnings.txt 2>&1" - $MVN clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/trunkJavadocWarnings.txt 2>&1 - if [[ $? != 0 ]] ; then - echo "Trunk javadoc compilation is broken?" - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 patch{color}. Trunk compilation may be broken." - return 1 - fi - - return 0 -} - -############################################################################### -### Check for @author tags in the patch -checkAuthor () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Checking there are no @author tags in the patch." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - authorTags=`$GREP -c -i '@author' $PATCH_DIR/patch` - echo "There appear to be $authorTags @author tags in the patch." - if [[ $authorTags != 0 ]] ; then - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 @author{color}. The patch appears to contain $authorTags @author tags which the Hadoop community has agreed to not allow in code contributions." - return 1 - fi - JIRA_COMMENT="$JIRA_COMMENT - - {color:green}+1 @author{color}. The patch does not contain any @author tags." - return 0 -} - -############################################################################### -### Check for tests in the patch -checkTests () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Checking there are new or changed tests in the patch." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - testReferences=`$GREP -c -i -e '^+++.*/test' $PATCH_DIR/patch` - echo "There appear to be $testReferences test files referenced in the patch." - if [[ $testReferences == 0 ]] ; then - if [[ $JENKINS == "true" ]] ; then - patchIsDoc=`$GREP -c -i 'title="documentation' $PATCH_DIR/jira` - if [[ $patchIsDoc != 0 ]] ; then - echo "The patch appears to be a documentation patch that doesn't require tests." - JIRA_COMMENT="$JIRA_COMMENT - - {color:green}+0 tests included{color}. The patch appears to be a documentation patch that doesn't require tests." - return 0 - fi - fi - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 tests included{color}. The patch doesn't appear to include any new or modified tests. - Please justify why no new tests are needed for this patch. - Also please list what manual steps were performed to verify this patch." - return 1 - fi - JIRA_COMMENT="$JIRA_COMMENT - - {color:green}+1 tests included{color}. The patch appears to include $testReferences new or modified test files." - return 0 -} - -cleanUpXml () { - cd $BASEDIR/conf - for file in `ls *.xml.template` - do - rm -f `basename $file .template` - done - cd $BASEDIR -} - -############################################################################### -### Attempt to apply the patch -applyPatch () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Applying patch." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" export PATCH - $bindir/smart-apply-patch.sh $PATCH_DIR/patch + "${BINDIR}/smart-apply-patch.sh" "${PATCH_DIR}/patch" if [[ $? != 0 ]] ; then echo "PATCH APPLICATION FAILED" - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 patch{color}. The patch command could not apply the patch." - return 1 + ((RESULT = RESULT + 1)) + add_jira_table -1 patch "The patch command could not apply the patch." + output_to_console 1 + output_to_jira 1 + cleanup_and_exit 1 fi return 0 } -############################################################################### -calculateJavadocWarnings() { - WARNING_FILE="$1" - RET=$(egrep "^[0-9]+ warnings$" "$WARNING_FILE" | awk '{sum+=$1} END {print sum}') -} -### Check there are no javadoc warnings -checkJavadocWarnings () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Determining number of patched javadoc warnings." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - echo "$MVN clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchJavadocWarnings.txt 2>&1" - if [ -d hadoop-project ]; then - (cd hadoop-project; $MVN install > /dev/null 2>&1) - fi - if [ -d hadoop-common-project/hadoop-annotations ]; then - (cd hadoop-common-project/hadoop-annotations; $MVN install > /dev/null 2>&1) - fi - $MVN clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchJavadocWarnings.txt 2>&1 - calculateJavadocWarnings "$PATCH_DIR/trunkJavadocWarnings.txt" - numTrunkJavadocWarnings=$RET - calculateJavadocWarnings "$PATCH_DIR/patchJavadocWarnings.txt" - numPatchJavadocWarnings=$RET - grep -i warning "$PATCH_DIR/trunkJavadocWarnings.txt" > "$PATCH_DIR/trunkJavadocWarningsFiltered.txt" - grep -i warning "$PATCH_DIR/patchJavadocWarnings.txt" > "$PATCH_DIR/patchJavadocWarningsFiltered.txt" - diff -u "$PATCH_DIR/trunkJavadocWarningsFiltered.txt" \ - "$PATCH_DIR/patchJavadocWarningsFiltered.txt" > \ - "$PATCH_DIR/diffJavadocWarnings.txt" - rm -f "$PATCH_DIR/trunkJavadocWarningsFiltered.txt" "$PATCH_DIR/patchJavadocWarningsFiltered.txt" - echo "There appear to be $numTrunkJavadocWarnings javadoc warnings before the patch and $numPatchJavadocWarnings javadoc warnings after applying the patch." - if [[ $numTrunkJavadocWarnings != "" && $numPatchJavadocWarnings != "" ]] ; then - if [[ $numPatchJavadocWarnings -gt $numTrunkJavadocWarnings ]] ; then - JIRA_COMMENT="$JIRA_COMMENT +## @description If this patches actually patches test-patch.sh, then +## @description run with the patched version for the test. +## @audience private +## @stability evolving +## @replaceable no +## @return none; otherwise relaunches +function check_reexec +{ + set +x + local commentfile=${PATCH_DIR}/tp.${RANDOM} - {color:red}-1 javadoc{color}. The javadoc tool appears to have generated `expr $(($numPatchJavadocWarnings-$numTrunkJavadocWarnings))` warning messages. - See $BUILD_URL/artifact/patchprocess/diffJavadocWarnings.txt for details." - return 1 + if [[ ${REEXECED} == true ]]; then + big_console_header "Re-exec mode detected. Continuing." + return + fi + + if [[ ! ${CHANGED_FILES} =~ dev-support/test-patch + || ${CHANGED_FILES} =~ dev-support/smart-apply ]] ; then + return + fi + + big_console_header "dev-support patch detected" + printf "\n\nRe-executing against patched versions to test.\n\n" + + apply_patch_file + + if [[ ${JENKINS} == true ]]; then + + rm "${commentfile}" 2>/dev/null + + echo "(!) A patch to test-patch or smart-apply-patch has been detected. " > "${commentfile}" + echo "Re-executing against the patched versions to perform further tests. " >> "${commentfile}" + echo "The console is at ${BUILD_URL}/console in case of problems." >> "${commentfile}" + + if [[ ${OFFLINE} == false ]]; then + export USER=hudson + # shellcheck disable=SC2086 + ${JIRACLI} --comment "$(cat ${commentfile})" \ + -s https://issues.apache.org/jira \ + -a addcomment -u hadoopqa \ + -p "${JIRA_PASSWD}" \ + --issue "${ISSUE}" + ${JIRACLI} -s https://issues.apache.org/jira \ + -a logout -u hadoopqa \ + -p "${JIRA_PASSWD}" + + rm "${commentfile}" fi fi - JIRA_COMMENT="$JIRA_COMMENT - {color:green}+1 javadoc{color}. There were no new javadoc warning messages." + cd "${CWD}" + mkdir -p "${PATCH_DIR}/dev-support-test" + cp -pr "${BASEDIR}"/dev-support/test-patch* "${PATCH_DIR}/dev-support-test" + cp -pr "${BASEDIR}"/dev-support/smart-apply* "${PATCH_DIR}/dev-support-test" + + big_console_header "exec'ing test-patch.sh now..." + + exec "${PATCH_DIR}/dev-support-test/test-patch.sh" \ + --reexec \ + --patch-dir="${PATCH_DIR}" \ + "${USER_PARAMS[@]}" +} + + + +## @description Check the current directory for @author tags +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function check_author +{ + local authorTags + + big_console_header "Checking there are no @author tags in the patch." + + start_clock + + authorTags=$("${GREP}" -c -i '@author' "${PATCH_DIR}/patch") + echo "There appear to be ${authorTags} @author tags in the patch." + if [[ ${authorTags} != 0 ]] ; then + add_jira_table -1 @author \ + "The patch appears to contain ${authorTags} @author tags which the Hadoop" \ + " community has agreed to not allow in code contributions." + return 1 + fi + add_jira_table +1 @author "The patch does not contain any @author tags." return 0 } -############################################################################### -### Check there are no changes in the number of Javac warnings -checkJavacWarnings () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Determining number of patched javac warnings." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - echo "$MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess $NATIVE_PROFILE -Ptest-patch > $PATCH_DIR/patchJavacWarnings.txt 2>&1" - $MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess $NATIVE_PROFILE -Ptest-patch > $PATCH_DIR/patchJavacWarnings.txt 2>&1 - if [[ $? != 0 ]] ; then - JIRA_COMMENT="$JIRA_COMMENT +## @description Check the patch file for changed/new tests +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function check_modified_unittests +{ + local testReferences=0 + local i - {color:red}-1 javac{color:red}. The patch appears to cause the build to fail." + verify_needed_test unit + + if [[ $? == 0 ]]; then + return 0 + fi + + big_console_header "Checking there are new or changed tests in the patch." + + start_clock + + for i in ${CHANGED_FILES}; do + if [[ ${i} =~ /test/ ]]; then + ((testReferences=testReferences + 1)) + fi + done + + echo "There appear to be ${testReferences} test file(s) referenced in the patch." + if [[ ${testReferences} == 0 ]] ; then + add_jira_table -1 "tests included" \ + "The patch doesn't appear to include any new or modified tests. " \ + "Please justify why no new tests are needed for this patch." \ + "Also please list what manual steps were performed to verify this patch." + return 1 + fi + add_jira_table +1 "tests included" \ + "The patch appears to include ${testReferences} new or modified test files." + return 0 +} + +## @description Helper for check_javadoc +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function count_javadoc_warns +{ + local warningfile=$1 + + #shellcheck disable=SC2016,SC2046 + return $(${EGREP} "^[0-9]+ warnings$" "${warningfile}" | ${AWK} '{sum+=$1} END {print sum}') +} + +## @description Count and compare the number of JavaDoc warnings pre- and post- patch +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function check_javadoc +{ + local numBranchJavadocWarnings + local numPatchJavadocWarnings + + verify_needed_test javadoc + + if [[ $? == 0 ]]; then + echo "This patch does not appear to need javadoc checks." + return 0 + fi + + big_console_header "Determining number of patched javadoc warnings" + + start_clock + + if [[ -d hadoop-project ]]; then + (cd hadoop-project; "${MVN}" install > /dev/null 2>&1) + fi + if [[ -d hadoop-common-project/hadoop-annotations ]]; then + (cd hadoop-common-project/hadoop-annotations; "${MVN}" install > /dev/null 2>&1) + fi + echo_and_redirect "${PATCH_DIR}/patchJavadocWarnings.txt" "${MVN}" clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess + count_javadoc_warns "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarnings.txt" + numBranchJavadocWarnings=$? + count_javadoc_warns "${PATCH_DIR}/patchJavadocWarnings.txt" + numPatchJavadocWarnings=$? + + echo "There appear to be ${numBranchJavadocWarnings} javadoc warnings before the patch and ${numPatchJavadocWarnings} javadoc warnings after applying the patch." + if [[ ${numBranchJavadocWarnings} != "" && ${numPatchJavadocWarnings} != "" ]] ; then + if [[ ${numPatchJavadocWarnings} -gt ${numBranchJavadocWarnings} ]] ; then + + ${GREP} -i warning "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarnings.txt" > "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarningsFiltered.txt" + ${GREP} -i warning "${PATCH_DIR}/patchJavadocWarnings.txt" > "${PATCH_DIR}/patchJavadocWarningsFiltered.txt" + ${DIFF} -u "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarningsFiltered.txt" \ + "${PATCH_DIR}/patchJavadocWarningsFiltered.txt" \ + > "${PATCH_DIR}/diffJavadocWarnings.txt" + rm -f "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarningsFiltered.txt" "${PATCH_DIR}/patchJavadocWarningsFiltered.txt" + + add_jira_table -1 javadoc "The applied patch generated "\ + "$((numPatchJavadocWarnings-numBranchJavadocWarnings))" \ + " additional warning messages." + add_jira_footer javadoc "@@BASE@@/diffJavadocWarnings.txt" + return 1 + fi + fi + add_jira_table +1 javadoc "There were no new javadoc warning messages." + return 0 +} + +## @description Make sure site still compiles +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function check_site +{ + local -r mypwd=$(pwd) + + verify_needed_test site + + if [[ $? == 0 ]]; then + echo "This patch does not appear to need site checks." + return 0 + fi + + big_console_header "Determining if patched site still builds" + + start_clock + + echo "site creation for ${mypwd}" + echo_and_redirect "${PATCH_DIR}/patchSiteWarnings.txt" "${MVN}" clean site site:stage -DskipTests -Dmaven.javadoc.skip=true -D${PROJECT_NAME}PatchProcess + if [[ $? != 0 ]] ; then + echo "Site compilation is broken" + add_jira_table -1 site "Site compilation is broken." + add_jira_footer site "@@BASE@@/patchSiteWarnings.txt" + return 1 + fi + add_jira_table +1 site "Site still builds." + return 0 +} + +## @description Helper for check_javac +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function count_javac_warns +{ + local warningfile=$1 + #shellcheck disable=SC2016,SC2046 + return $(${AWK} 'BEGIN {total = 0} {total += 1} END {print total}' "${warningfile}") +} + +## @description Count and compare the number of javac warnings pre- and post- patch +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function check_javac +{ + local branchJavacWarnings + local patchJavacWarnings + + verify_needed_test javac + + if [[ $? == 0 ]]; then + echo "This patch does not appear to need javac checks." + return 0 + fi + + big_console_header "Determining number of patched javac warnings." + + start_clock + + echo_and_redirect "${PATCH_DIR}/patchJavacWarnings.txt" "${MVN}" clean test -DskipTests -D${PROJECT_NAME}PatchProcess "${NATIVE_PROFILE}" -Ptest-patch + if [[ $? != 0 ]] ; then + add_jira_table -1 javac "The patch appears to cause the build to fail." return 2 fi - ### Compare trunk and patch javac warning numbers - if [[ -f $PATCH_DIR/patchJavacWarnings.txt ]] ; then - $GREP '\[WARNING\]' $PATCH_DIR/trunkJavacWarnings.txt > $PATCH_DIR/filteredTrunkJavacWarnings.txt - $GREP '\[WARNING\]' $PATCH_DIR/patchJavacWarnings.txt > $PATCH_DIR/filteredPatchJavacWarnings.txt - trunkJavacWarnings=`cat $PATCH_DIR/filteredTrunkJavacWarnings.txt | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'` - patchJavacWarnings=`cat $PATCH_DIR/filteredPatchJavacWarnings.txt | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'` - echo "There appear to be $trunkJavacWarnings javac compiler warnings before the patch and $patchJavacWarnings javac compiler warnings after applying the patch." - if [[ $patchJavacWarnings != "" && $trunkJavacWarnings != "" ]] ; then - if [[ $patchJavacWarnings -gt $trunkJavacWarnings ]] ; then - JIRA_COMMENT="$JIRA_COMMENT + ### Compare ${PATCH_BRANCH} and patch javac warning numbers + if [[ -f ${PATCH_DIR}/patchJavacWarnings.txt ]] ; then + ${GREP} '\[WARNING\]' "${PATCH_DIR}/${PATCH_BRANCH}JavacWarnings.txt" > "${PATCH_DIR}/filtered${PATCH_BRANCH}JavacWarnings.txt" + ${GREP} '\[WARNING\]' "${PATCH_DIR}/patchJavacWarnings.txt" > "${PATCH_DIR}/filteredPatchJavacWarnings.txt" - {color:red}-1 javac{color}. The applied patch generated $patchJavacWarnings javac compiler warnings (more than the trunk's current $trunkJavacWarnings warnings)." + count_javac_warns "${PATCH_DIR}/filtered${PATCH_BRANCH}JavacWarnings.txt" + branchJavacWarnings=$? + count_javac_warns "${PATCH_DIR}/filteredPatchJavacWarnings.txt" + patchJavacWarnings=$? - $DIFF $PATCH_DIR/filteredTrunkJavacWarnings.txt $PATCH_DIR/filteredPatchJavacWarnings.txt > $PATCH_DIR/diffJavacWarnings.txt - JIRA_COMMENT_FOOTER="Javac warnings: $BUILD_URL/artifact/patchprocess/diffJavacWarnings.txt -$JIRA_COMMENT_FOOTER" + echo "There appear to be ${branchJavacWarnings} javac compiler warnings before the patch and ${patchJavacWarnings} javac compiler warnings after applying the patch." + if [[ ${patchJavacWarnings} != "" && ${branchJavacWarnings} != "" ]] ; then + if [[ ${patchJavacWarnings} -gt ${branchJavacWarnings} ]] ; then + + ${DIFF} "${PATCH_DIR}/filtered${PATCH_BRANCH}JavacWarnings.txt" \ + "${PATCH_DIR}/filteredPatchJavacWarnings.txt" \ + > "${PATCH_DIR}/diffJavacWarnings.txt" + + add_jira_table -1 javac "The applied patch generated "\ + "$((patchJavacWarnings-${PATCH_BRANCH}JavacWarnings))" \ + " additional warning messages." + + add_jira_footer javac "@@BASE@@/diffJavacWarnings.txt" return 1 fi fi fi - JIRA_COMMENT="$JIRA_COMMENT - {color:green}+1 javac{color}. The applied patch does not increase the total number of javac compiler warnings." + add_jira_table +1 javac "There were no new javac warning messages." return 0 } -############################################################################### -### Check there are no changes in the number of release audit (RAT) warnings -checkReleaseAuditWarnings () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Determining number of patched release audit warnings." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - echo "$MVN apache-rat:check -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchReleaseAuditOutput.txt 2>&1" - $MVN apache-rat:check -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchReleaseAuditOutput.txt 2>&1 - find $BASEDIR -name rat.txt | xargs cat > $PATCH_DIR/patchReleaseAuditWarnings.txt +## @description Verify all files have an Apache License +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function check_apachelicense +{ + big_console_header "Determining number of patched release audit warnings." - ### Compare trunk and patch release audit warning numbers - if [[ -f $PATCH_DIR/patchReleaseAuditWarnings.txt ]] ; then - patchReleaseAuditWarnings=`$GREP -c '\!?????' $PATCH_DIR/patchReleaseAuditWarnings.txt` + start_clock + + echo_and_redirect "${PATCH_DIR}/patchReleaseAuditOutput.txt" "${MVN}" apache-rat:check -D${PROJECT_NAME}PatchProcess + #shellcheck disable=SC2038 + find "${BASEDIR}" -name rat.txt | xargs cat > "${PATCH_DIR}/patchReleaseAuditWarnings.txt" + + ### Compare ${PATCH_BRANCH} and patch release audit warning numbers + if [[ -f ${PATCH_DIR}/patchReleaseAuditWarnings.txt ]] ; then + patchReleaseAuditWarnings=$("${GREP}" -c '\!?????' "${PATCH_DIR}/patchReleaseAuditWarnings.txt") echo "" echo "" - echo "There appear to be $patchReleaseAuditWarnings release audit warnings after applying the patch." - if [[ $patchReleaseAuditWarnings != "" ]] ; then - if [[ $patchReleaseAuditWarnings -gt 0 ]] ; then - JIRA_COMMENT="$JIRA_COMMENT + echo "There appear to be ${patchReleaseAuditWarnings} release audit warnings after applying the patch." + if [[ ${patchReleaseAuditWarnings} != "" ]] ; then + if [[ ${patchReleaseAuditWarnings} -gt 0 ]] ; then + add_jira_table -1 "release audit" "The applied patch generated ${patchReleaseAuditWarnings} release audit warnings." + + ${GREP} '\!?????' "${PATCH_DIR}/patchReleaseAuditWarnings.txt" \ + > "${PATCH_DIR}/patchReleaseAuditProblems.txt" + + echo "Lines that start with ????? in the release audit "\ + "report indicate files that do not have an Apache license header." \ + >> "${PATCH_DIR}/patchReleaseAuditProblems.txt" + + add_jira_footer "Release Audit" "@@BASE@@/patchReleaseAuditProblems.txt" - {color:red}-1 release audit{color}. The applied patch generated $patchReleaseAuditWarnings release audit warnings." - $GREP '\!?????' $PATCH_DIR/patchReleaseAuditWarnings.txt > $PATCH_DIR/patchReleaseAuditProblems.txt - echo "Lines that start with ????? in the release audit report indicate files that do not have an Apache license header." >> $PATCH_DIR/patchReleaseAuditProblems.txt - JIRA_COMMENT_FOOTER="Release audit warnings: $BUILD_URL/artifact/patchprocess/patchReleaseAuditProblems.txt -$JIRA_COMMENT_FOOTER" return 1 fi fi fi - JIRA_COMMENT="$JIRA_COMMENT - - {color:green}+1 release audit{color}. The applied patch does not increase the total number of release audit warnings." + add_jira_table 1 "release audit" "The applied patch does not increase the total number of release audit warnings." return 0 } -############################################################################### -### Check there are no changes in the number of Checkstyle warnings -checkStyle () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Determining number of patched checkstyle warnings." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - echo "THIS IS NOT IMPLEMENTED YET" - echo "" - echo "" - echo "$MVN test checkstyle:checkstyle -DskipTests -D${PROJECT_NAME}PatchProcess" - $MVN test checkstyle:checkstyle -DskipTests -D${PROJECT_NAME}PatchProcess +## @description Verify mvn install works +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function check_mvn_install +{ + local retval - JIRA_COMMENT_FOOTER="Checkstyle results: $BUILD_URL/artifact/trunk/build/test/checkstyle-errors.html -$JIRA_COMMENT_FOOTER" - ### TODO: calculate actual patchStyleErrors -# patchStyleErrors=0 -# if [[ $patchStyleErrors != 0 ]] ; then -# JIRA_COMMENT="$JIRA_COMMENT -# -# {color:red}-1 checkstyle{color}. The patch generated $patchStyleErrors code style errors." -# return 1 -# fi -# JIRA_COMMENT="$JIRA_COMMENT -# -# {color:green}+1 checkstyle{color}. The patch generated 0 code style errors." - return 0 + verify_needed_test javadoc + retval=$? + + verify_needed_test javac + ((retval = retval + $? )) + + if [[ ${retval} == 0 ]]; then + echo "This patch does not appear to need mvn install checks." + return 0 + fi + + big_console_header "Installing all of the jars" + + start_clock + echo_and_redirect "${PATCH_DIR}/jarinstall.txt" "${MVN}" install -Dmaven.javadoc.skip=true -DskipTests -D${PROJECT_NAME}PatchProcess + retval=$? + if [[ ${retval} != 0 ]]; then + add_jira_table -1 install "The patch causes mvn install to fail." + else + add_jira_table +1 install "mvn install still works." + fi + return ${retval} } -############################################################################### -### Install the new jars so tests and findbugs can find all of the updated jars -buildAndInstall () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Installing all of the jars" - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - echo "$MVN install -Dmaven.javadoc.skip=true -DskipTests -D${PROJECT_NAME}PatchProcess" - $MVN install -Dmaven.javadoc.skip=true -DskipTests -D${PROJECT_NAME}PatchProcess - return $? -} +## @description Verify patch does not trigger any findbugs warnings +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function check_findbugs +{ + local findbugs_version + local modules=${CHANGED_MODULES} + local rc=0 + local module_suffix + local findbugsWarnings=0 + local relative_file + local newFindbugsWarnings + local findbugsWarnings + local line + local firstpart + local secondpart + + big_console_header "Determining number of patched Findbugs warnings." -############################################################################### -### Check there are no changes in the number of Findbugs warnings -checkFindbugsWarnings () { - findbugs_version=`${FINDBUGS_HOME}/bin/findbugs -version` - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Determining number of patched Findbugs warnings." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - - modules=$(findModules) - rc=0 - for module in $modules; + verify_needed_test findbugs + if [[ $? == 0 ]]; then + echo "Patch does not touch any java files. Skipping findbugs." + return 0 + fi + + start_clock + + if [[ ! -e "${FINDBUGS_HOME}/bin/findbugs" ]]; then + printf "\n\n%s is not executable.\n\n" "${FINDBUGS_HOME}/bin/findbugs" + add_jira_table -1 findbugs "Findbugs is not installed." + return 1 + fi + + findbugs_version=$("${FINDBUGS_HOME}/bin/findbugs" -version) + + for module in ${modules} do - cd $module - echo " Running findbugs in $module" - module_suffix=`basename ${module}` - echo "$MVN clean test findbugs:findbugs -DskipTests -D${PROJECT_NAME}PatchProcess < /dev/null > $PATCH_DIR/patchFindBugsOutput${module_suffix}.txt 2>&1" - $MVN clean test findbugs:findbugs -DskipTests -D${PROJECT_NAME}PatchProcess < /dev/null > $PATCH_DIR/patchFindBugsOutput${module_suffix}.txt 2>&1 + pushd "${module}" >/dev/null + echo " Running findbugs in ${module}" + module_suffix=$(basename "${module}") + echo_and_redirect "${PATCH_DIR}/patchFindBugsOutput${module_suffix}.txt" "${MVN}" clean test findbugs:findbugs -DskipTests -D${PROJECT_NAME}PatchProcess \ + < /dev/null (( rc = rc + $? )) - cd - + popd >/dev/null done - if [ $rc != 0 ] ; then - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 findbugs{color}. The patch appears to cause Findbugs (version ${findbugs_version}) to fail." + if [[ ${rc} -ne 0 ]]; then + add_jira_table -1 findbugs "The patch appears to cause Findbugs (version ${findbugs_version}) to fail." return 1 fi - - findbugsWarnings=0 - for file in $(find $BASEDIR -name findbugsXml.xml) + + while read file do - relative_file=${file#$BASEDIR/} # strip leading $BASEDIR prefix - if [ ! $relative_file == "target/findbugsXml.xml" ]; then + relative_file=${file#${BASEDIR}/} # strip leading ${BASEDIR} prefix + if [[ ${relative_file} != "target/findbugsXml.xml" ]]; then module_suffix=${relative_file%/target/findbugsXml.xml} # strip trailing path - module_suffix=`basename ${module_suffix}` + module_suffix=$(basename "${module_suffix}") fi - - cp $file $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml - $FINDBUGS_HOME/bin/setBugDatabaseInfo -timestamp "01/01/2000" \ - $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml \ - $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml - newFindbugsWarnings=`$FINDBUGS_HOME/bin/filterBugs -first "01/01/2000" $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml \ - $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml | $AWK '{print $1}'` + + cp "${file}" "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml" + + "${FINDBUGS_HOME}/bin/setBugDatabaseInfo" -timestamp "01/01/2000" \ + "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml" \ + "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml" + + newFindbugsWarnings=$("${FINDBUGS_HOME}/bin/filterBugs" \ + -first "01/01/2000" "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml" \ + "${PATCH_DIR}/newPatchFindbugsWarnings${module_suffix}.xml" \ + | ${AWK} '{print $1}') + echo "Found $newFindbugsWarnings Findbugs warnings ($file)" + findbugsWarnings=$((findbugsWarnings+newFindbugsWarnings)) - $FINDBUGS_HOME/bin/convertXmlToText -html \ - $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml \ - $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.html - if [[ $newFindbugsWarnings > 0 ]] ; then - JIRA_COMMENT_FOOTER="Findbugs warnings: $BUILD_URL/artifact/patchprocess/newPatchFindbugsWarnings${module_suffix}.html -$JIRA_COMMENT_FOOTER" + + "${FINDBUGS_HOME}/bin/convertXmlToText" -html \ + "${PATCH_DIR}/newPatchFindbugsWarnings${module_suffix}.xml" \ + "${PATCH_DIR}/newPatchFindbugsWarnings${module_suffix}.html" + + if [[ ${newFindbugsWarnings} -gt 0 ]] ; then + populate_test_table FindBugs "module:${module_suffix}" + while read line; do + firstpart=$(echo "${line}" | cut -f2 -d:) + secondpart=$(echo "${line}" | cut -f9- -d' ') + add_jira_test_table "" "${firstpart}:${secondpart}" + done < <("${FINDBUGS_HOME}/bin/convertXmlToText" \ + "${PATCH_DIR}/newPatchFindbugsWarnings${module_suffix}.xml") + + add_jira_footer "Findbugs warnings" "@@BASE@@/newPatchFindbugsWarnings${module_suffix}.html" + fi + done < <(find "${BASEDIR}" -name findbugsXml.xml) + + if [[ ${findbugsWarnings} -gt 0 ]] ; then + add_jira_table -1 findbugs "The patch appears to introduce ${findbugsWarnings} new Findbugs (version ${findbugs_version}) warnings." + return 1 + fi + + add_jira_table +1 findbugs "The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings." + return 0 +} + +## @description Make sure Maven's eclipse generation works. +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function check_mvn_eclipse +{ + big_console_header "Running mvn eclipse:eclipse." + + verify_needed_test javac + if [[ $? == 0 ]]; then + echo "Patch does not touch any java files. Skipping mvn eclipse:eclipse" + return 0 + fi + + start_clock + + echo_and_redirect "${PATCH_DIR}/patchEclipseOutput.txt" "${MVN}" eclipse:eclipse -D${PROJECT_NAME}PatchProcess + if [[ $? != 0 ]] ; then + add_jira_table -1 eclipse:eclipse "The patch failed to build with eclipse:eclipse." + return 1 + fi + add_jira_table +1 eclipse:eclipse "The patch built with eclipse:eclipse." + return 0 +} + +## @description Utility to push many tests into the failure list +## @audience private +## @stability evolving +## @replaceable no +## @param testdesc +## @param testlist +function populate_test_table +{ + local reason=$1 + shift + local first="" + local i + + for i in "$@"; do + if [[ -z "${first}" ]]; then + add_jira_test_table "${reason}" "${i}" + first="${reason}" + else + add_jira_test_table " " "${i}" fi done - - if [[ $findbugsWarnings -gt 0 ]] ; then - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 findbugs{color}. The patch appears to introduce $findbugsWarnings new Findbugs (version ${findbugs_version}) warnings." - return 1 - fi - JIRA_COMMENT="$JIRA_COMMENT - - {color:green}+1 findbugs{color}. The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings." - return 0 } -############################################################################### -### Verify eclipse:eclipse works -checkEclipseGeneration () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Running mvn eclipse:eclipse." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" +## @description Run and verify the output of the appropriate unit tests +## @audience private +## @stability evolving +## @replaceable no +## @return 0 on success +## @return 1 on failure +function check_unittests +{ + verify_needed_test unit - echo "$MVN eclipse:eclipse -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchEclipseOutput.txt 2>&1" - $MVN eclipse:eclipse -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchEclipseOutput.txt 2>&1 - if [[ $? != 0 ]] ; then - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 eclipse:eclipse{color}. The patch failed to build with eclipse:eclipse." - return 1 + if [[ $? == 0 ]]; then + echo "Existing unit tests do not test patched files. Skipping." + return 0 fi - JIRA_COMMENT="$JIRA_COMMENT - {color:green}+1 eclipse:eclipse{color}. The patch built with eclipse:eclipse." - return 0 -} + big_console_header "Running unit tests" + start_clock + local failed_tests="" + local modules=${CHANGED_MODULES} + local building_common=0 + local hdfs_modules + local ordered_modules="" + local failed_test_builds="" + local test_timeouts="" + local test_logfile + local test_build_result + local module_test_timeouts="" + local result + local totalresult=0 + local module_prefix -############################################################################### -### Run the tests -runTests () { - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Running tests." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - - failed_tests="" - modules=$(findModules) # # If we are building hadoop-hdfs-project, we must build the native component # of hadoop-common-project first. In order to accomplish this, we move the @@ -733,271 +1745,570 @@ runTests () { # explicitly insert a mvn compile -Pnative of common, to ensure that the # native libraries show up where we need them. # - building_common=0 - for module in $modules; do - if [[ $module == hadoop-hdfs-project* ]]; then - hdfs_modules="$hdfs_modules $module" - elif [[ $module == hadoop-common-project* ]]; then - ordered_modules="$ordered_modules $module" - building_common=1 + + for module in ${modules}; do + if [[ ${module} == hadoop-hdfs-project* ]]; then + hdfs_modules="${hdfs_modules} ${module}" + elif [[ ${module} == hadoop-common-project* ]]; then + ordered_modules="${ordered_modules} ${module}" + building_common=1 + else + ordered_modules="${ordered_modules} ${module}" + fi + done + + if [[ -n "${hdfs_modules}" ]]; then + ordered_modules="${ordered_modules} ${hdfs_modules}" + if [[ ${building_common} -eq 0 ]]; then + echo " Building hadoop-common with -Pnative in order to provide libhadoop.so to the hadoop-hdfs unit tests." + echo_and_redirect "${PATCH_DIR}/testrun_native.txt" "${MVN}" compile "${NATIVE_PROFILE}" "-D${PROJECT_NAME}PatchProcess" + if [[ $? != 0 ]]; then + add_jira_table -1 "native" "Failed to build the native portion " \ + "of hadoop-common prior to running the unit tests in ${ordered_modules}" + return 1 else - ordered_modules="$ordered_modules $module" - fi - done - if [ -n "$hdfs_modules" ]; then - ordered_modules="$ordered_modules $hdfs_modules" - if [[ $building_common -eq 0 ]]; then - echo " Building hadoop-common with -Pnative in order to provide \ -libhadoop.so to the hadoop-hdfs unit tests." - echo " $MVN compile $NATIVE_PROFILE -D${PROJECT_NAME}PatchProcess" - if ! $MVN compile $NATIVE_PROFILE -D${PROJECT_NAME}PatchProcess; then - JIRA_COMMENT="$JIRA_COMMENT - {color:red}-1 core tests{color}. Failed to build the native portion \ -of hadoop-common prior to running the unit tests in $ordered_modules" - return 1 - fi + add_jira_table +1 "native" "Pre-build of native portion" fi + fi fi - failed_test_builds="" - test_timeouts="" - for module in $ordered_modules; do - cd $module - module_suffix=`basename ${module}` - test_logfile=$PATCH_DIR/testrun_${module_suffix}.txt - echo " Running tests in $module" - echo " $MVN clean install -fn $NATIVE_PROFILE $REQUIRE_TEST_LIB_HADOOP -D${PROJECT_NAME}PatchProcess" - $MVN clean install -fae $NATIVE_PROFILE $REQUIRE_TEST_LIB_HADOOP -D${PROJECT_NAME}PatchProcess > $test_logfile 2>&1 + + for module in ${ordered_modules}; do + result=0 + start_clock + pushd "${module}" >/dev/null + module_suffix=$(basename "${module}") + module_prefix=$(echo "${module}" | cut -f2 -d- ) + + test_logfile=${PATCH_DIR}/testrun_${module_suffix}.txt + echo " Running tests in ${module_suffix}" + echo_and_redirect "${test_logfile}" "${MVN}" clean install -fae "${NATIVE_PROFILE}" "${REQUIRE_TEST_LIB_HADOOP}" -D${PROJECT_NAME}PatchProcess test_build_result=$? - cat $test_logfile - module_test_timeouts=`$AWK '/^Running / { if (last) { print last } last=$2 } /^Tests run: / { last="" }' $test_logfile` - if [[ -n "$module_test_timeouts" ]] ; then - test_timeouts="$test_timeouts -$module_test_timeouts" + + add_jira_footer "${module_suffix} test log" "@@BASE@@/testrun_${module_suffix}.txt" + + # shellcheck disable=2016 + module_test_timeouts=$(${AWK} '/^Running / { if (last) { print last } last=$2 } /^Tests run: / { last="" }' "${test_logfile}") + if [[ -n "${module_test_timeouts}" ]] ; then + test_timeouts="${test_timeouts} ${module_test_timeouts}" + result=1 fi - module_failed_tests=`find . -name 'TEST*.xml' | xargs $GREP -l -E "/dev/null + + if [[ $result == 1 ]]; then + add_jira_table -1 "${module_prefix} tests" "Tests failed in ${module_suffix}." + else + add_jira_table +1 "${module_prefix} tests" "Tests passed in ${module_suffix}." + fi + + ((totalresult = totalresult + result)) done - result=0 - comment_prefix=" {color:red}-1 core tests{color}." - if [[ -n "$failed_tests" ]] ; then - JIRA_COMMENT="$JIRA_COMMENT -$comment_prefix The patch failed these unit tests in $modules: -$failed_tests" - comment_prefix=" " - result=1 + if [[ -n "${failed_tests}" ]] ; then + # shellcheck disable=SC2086 + populate_test_table "Failed unit tests" ${failed_tests} fi - if [[ -n "$test_timeouts" ]] ; then - JIRA_COMMENT="$JIRA_COMMENT - -$comment_prefix The following test timeouts occurred in $modules: -$test_timeouts" - comment_prefix=" " - result=1 + if [[ -n "${test_timeouts}" ]] ; then + # shellcheck disable=SC2086 + populate_test_table "Timed out tests" ${test_timeouts} fi - if [[ -n "$failed_test_builds" ]] ; then - JIRA_COMMENT="$JIRA_COMMENT - -$comment_prefix The test build failed in $failed_test_builds" - result=1 + if [[ -n "${failed_test_builds}" ]] ; then + # shellcheck disable=SC2086 + populate_test_table "Failed build" ${failed_test_builds} fi - if [[ $result == 0 ]] ; then - JIRA_COMMENT="$JIRA_COMMENT - {color:green}+1 core tests{color}. The patch passed unit tests in $modules." + if [[ ${JENKINS} == true ]]; then + add_jira_footer "Test Results" "${BUILD_URL}testReport/" fi - return $result -} -############################################################################### -# Find the maven module containing the given file. -findModule (){ - dir=`dirname $1` - while [ 1 ] - do - if [ -f "$dir/pom.xml" ] - then - echo $dir - return + if [[ ${totalresult} -gt 0 ]]; then + return 1 else - dir=`dirname $dir` + return 0 fi - done } -findModules () { - # Come up with a list of changed files into $TMP - TMP=/tmp/tmp.paths.$$ - $GREP '^+++ \|^--- ' $PATCH_DIR/patch | cut -c '5-' | $GREP -v /dev/null | sort | uniq > $TMP - - # if all of the lines start with a/ or b/, then this is a git patch that - # was generated without --no-prefix - if ! $GREP -qv '^a/\|^b/' $TMP ; then - sed -i -e 's,^[ab]/,,' $TMP - fi - - # Now find all the modules that were changed - TMP_MODULES=/tmp/tmp.modules.$$ - for file in $(cut -f 1 $TMP | sort | uniq); do - echo $(findModule $file) >> $TMP_MODULES - done - rm $TMP - - # Filter out modules without code - CHANGED_MODULES="" - for module in $(cat $TMP_MODULES | sort | uniq); do - $GREP "pom" $module/pom.xml > /dev/null - if [ "$?" != 0 ]; then - CHANGED_MODULES="$CHANGED_MODULES $module" - fi - done - rm $TMP_MODULES - echo $CHANGED_MODULES -} -############################################################################### -### Submit a comment to the defect's Jira -submitJiraComment () { +## @description Print out the finished details on the console +## @audience private +## @stability evolving +## @replaceable no +## @param runresult +## @return 0 on success +## @return 1 on failure +function output_to_console +{ local result=$1 - ### Do not output the value of JIRA_COMMENT_FOOTER when run by a developer - if [[ $JENKINS == "false" ]] ; then - JIRA_COMMENT_FOOTER="" - fi - if [[ $result == 0 ]] ; then - comment="{color:green}+1 overall{color}. $JIRA_COMMENT + shift + local i + local ourstring + local vote + local subs + local ela + local comment + local commentfile1="${PATCH_DIR}/comment.1" + local commentfile2="${PATCH_DIR}/comment.2" + local normaltop + local line + local seccoladj=0 + local spcfx=${PATCH_DIR}/spcl.txt -$JIRA_COMMENT_FOOTER" + if [[ ${result} == 0 ]]; then + if [[ ${JENKINS} == false ]]; then + { + printf "IF9fX19fX19fX18gCjwgU3VjY2VzcyEgPgogLS0tLS0tLS0tLSAKIFwgICAg"; + printf "IC9cICBfX18gIC9cCiAgXCAgIC8vIFwvICAgXC8gXFwKICAgICAoKCAgICBP"; + printf "IE8gICAgKSkKICAgICAgXFwgLyAgICAgXCAvLwogICAgICAgXC8gIHwgfCAg"; + printf "XC8gCiAgICAgICAgfCAgfCB8ICB8ICAKICAgICAgICB8ICB8IHwgIHwgIAog"; + printf "ICAgICAgIHwgICBvICAgfCAgCiAgICAgICAgfCB8ICAgfCB8ICAKICAgICAg"; + printf "ICB8bXwgICB8bXwgIAo" + } > "${spcfx}" + fi + printf "\n\n+1 overall\n\n" else - comment="{color:red}-1 overall{color}. $JIRA_COMMENT - -$JIRA_COMMENT_FOOTER" + if [[ ${JENKINS} == false ]]; then + { + printf "IF9fX19fICAgICBfIF8gICAgICAgICAgICAgICAgXyAKfCAgX19ffF8gXyhf"; + printf "KSB8XyAgIF8gXyBfXyBfX198IHwKfCB8XyAvIF9gIHwgfCB8IHwgfCB8ICdf"; + printf "Xy8gXyBcIHwKfCAgX3wgKF98IHwgfCB8IHxffCB8IHwgfCAgX18vX3wKfF98"; + printf "ICBcX18sX3xffF98XF9fLF98X3wgIFxfX18oXykKICAgICAgICAgICAgICAg"; + printf "ICAgICAgICAgICAgICAgICAK" + } > "${spcfx}" + fi + printf "\n\n-1 overall\n\n" fi - ### Output the test result to the console - echo " + + if [[ -f ${spcfx} ]]; then + if which base64 >/dev/null 2>&1; then + base64 --decode "${spcfx}" 2>/dev/null + elif which openssl >/dev/null 2>&1; then + openssl enc -A -d -base64 -in "${spcfx}" 2>/dev/null + fi + echo + echo + rm "${spcfx}" + fi + + seccoladj=$(findlargest 2 "${JIRA_COMMENT_TABLE[@]}") + if [[ ${seccoladj} -lt 10 ]]; then + seccoladj=10 + fi + + seccoladj=$((seccoladj + 2 )) + i=0 + until [[ $i -eq ${#JIRA_HEADER[@]} ]]; do + printf "%s\n" "${JIRA_HEADER[${i}]}" + ((i=i+1)) + done + + printf "| %s | %*s | %s | %s\n" "Vote" ${seccoladj} Subsystem Runtime "Comment" + echo "============================================================================" + i=0 + until [[ $i -eq ${#JIRA_COMMENT_TABLE[@]} ]]; do + ourstring=$(echo "${JIRA_COMMENT_TABLE[${i}]}" | tr -s ' ') + vote=$(echo "${ourstring}" | cut -f2 -d\|) + vote=$(colorstripper "${vote}") + subs=$(echo "${ourstring}" | cut -f3 -d\|) + ela=$(echo "${ourstring}" | cut -f4 -d\|) + comment=$(echo "${ourstring}" | cut -f5 -d\|) + + echo "${comment}" | fold -s -w $((78-seccoladj-22)) > "${commentfile1}" + normaltop=$(head -1 "${commentfile1}") + ${SED} -e '1d' "${commentfile1}" > "${commentfile2}" + + printf "| %4s | %*s | %-10s |%-s\n" "${vote}" ${seccoladj} \ + "${subs}" "${ela}" "${normaltop}" + while read line; do + printf "| | %*s | | %-s\n" ${seccoladj} " " "${line}" + done < "${commentfile2}" + + ((i=i+1)) + rm "${commentfile2}" "${commentfile1}" 2>/dev/null + done + + if [[ ${#JIRA_TEST_TABLE[@]} -gt 0 ]]; then + seccoladj=$(findlargest 1 "${JIRA_TEST_TABLE[@]}") + printf "\n\n%*s | Tests\n" "${seccoladj}" "Reason" + i=0 + until [[ $i -eq ${#JIRA_TEST_TABLE[@]} ]]; do + ourstring=$(echo "${JIRA_TEST_TABLE[${i}]}" | tr -s ' ') + vote=$(echo "${ourstring}" | cut -f2 -d\|) + subs=$(echo "${ourstring}" | cut -f3 -d\|) + printf "%*s | %s\n" "${seccoladj}" "${vote}" "${subs}" + ((i=i+1)) + done + fi + + printf "\n\n|| Subsystem || Report/Notes ||\n" + echo "============================================================================" + i=0 + + until [[ $i -eq ${#JIRA_FOOTER_TABLE[@]} ]]; do + comment=$(echo "${JIRA_FOOTER_TABLE[${i}]}" | + ${SED} -e "s,@@BASE@@,${PATCH_DIR},g") + printf "%s\n" "${comment}" + ((i=i+1)) + done +} -$comment" +## @description Print out the finished details to the JIRA issue +## @audience private +## @stability evolving +## @replaceable no +## @param runresult +function output_to_jira +{ + local result=$1 + local i + local commentfile=${PATCH_DIR}/commentfile + local comment - if [[ $JENKINS == "true" ]] ; then - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Adding comment to Jira." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - ### Update Jira with a comment + rm "${commentfile}" 2>/dev/null + + if [[ ${JENKINS} != "true" ]] ; then + return 0 + fi + + big_console_header "Adding comment to JIRA" + + add_jira_footer "Console output" "${BUILD_URL}/console" + + if [[ ${result} == 0 ]]; then + add_jira_header "(/) *{color:green}+1 overall{color}*" + else + add_jira_header "(x) *{color:red}-1 overall{color}*" + fi + + + { echo "\\\\" ; echo "\\\\"; } >> "${commentfile}" + + i=0 + until [[ $i -eq ${#JIRA_HEADER[@]} ]]; do + printf "%s\n" "${JIRA_HEADER[${i}]}" >> "${commentfile}" + ((i=i+1)) + done + + { echo "\\\\" ; echo "\\\\"; } >> "${commentfile}" + + echo "|| Vote || Subsystem || Runtime || Comment ||" >> "${commentfile}" + + i=0 + until [[ $i -eq ${#JIRA_COMMENT_TABLE[@]} ]]; do + printf "%s\n" "${JIRA_COMMENT_TABLE[${i}]}" >> "${commentfile}" + ((i=i+1)) + done + + + if [[ ${#JIRA_TEST_TABLE[@]} -gt 0 ]]; then + { echo "\\\\" ; echo "\\\\"; } >> "${commentfile}" + + echo "|| Reason || Tests ||" >> "${commentfile}" + i=0 + until [[ $i -eq ${#JIRA_TEST_TABLE[@]} ]]; do + printf "%s\n" "${JIRA_TEST_TABLE[${i}]}" >> "${commentfile}" + ((i=i+1)) + done + fi + + { echo "\\\\" ; echo "\\\\"; } >> "${commentfile}" + + echo "|| Subsystem || Report/Notes ||" >> "${commentfile}" + i=0 + until [[ $i -eq ${#JIRA_FOOTER_TABLE[@]} ]]; do + comment=$(echo "${JIRA_FOOTER_TABLE[${i}]}" | + ${SED} -e "s,@@BASE@@,${BUILD_URL}artifact/patchprocess,g") + printf "%s\n" "${comment}" >> "${commentfile}" + ((i=i+1)) + done + + printf "\n\nThis message was automatically generated.\n\n" >> "${commentfile}" + + if [[ ${OFFLINE} == false ]]; then export USER=hudson - $JIRACLI -s https://issues.apache.org/jira -a addcomment -u hadoopqa -p $JIRA_PASSWD --comment "$comment" --issue $defect - $JIRACLI -s https://issues.apache.org/jira -a logout -u hadoopqa -p $JIRA_PASSWD + # shellcheck disable=SC2086 + ${JIRACLI} --comment "$(cat "${commentfile}")" \ + -s https://issues.apache.org/jira \ + -a addcomment -u hadoopqa \ + -p "${JIRA_PASSWD}" \ + --issue "${ISSUE}" + ${JIRACLI} -s https://issues.apache.org/jira \ + -a logout -u hadoopqa \ + -p "${JIRA_PASSWD}" fi } -############################################################################### -### Cleanup files -cleanupAndExit () { +## @description Clean the filesystem as appropriate and then exit +## @audience private +## @stability evolving +## @replaceable no +## @param runresult +function cleanup_and_exit +{ local result=$1 - if [[ $JENKINS == "true" ]] ; then - if [ -e "$PATCH_DIR" ] ; then - mv $PATCH_DIR $BASEDIR + + if [[ ${JENKINS} == "true" ]] ; then + if [[ -e "${PATCH_DIR}" ]] ; then + hadoop_debug "mv ${PATCH_DIR} ${BASEDIR} " + mv "${PATCH_DIR}" "${BASEDIR}" fi fi - echo "" - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Finished build." - echo "======================================================================" - echo "======================================================================" - echo "" - echo "" - exit $result + big_console_header "Finished build." + + # shellcheck disable=SC2086 + exit ${result} +} + +## @description Driver to execute _postcheckout routines +## @audience private +## @stability evolving +## @replaceable no +function postcheckout +{ + local routine + local plugin + + for routine in find_java_home verify_patch_file + do + hadoop_debug "Running ${routine}" + ${routine} + + (( RESULT = RESULT + $? )) + if [[ ${RESULT} != 0 ]] ; then + output_to_console 1 + output_to_jira 1 + cleanup_and_exit 1 + fi + done + + for plugin in ${PLUGINS}; do + if declare -f ${plugin}_postcheckout >/dev/null 2>&1; then + + hadoop_debug "Running ${plugin}_postcheckout" + #shellcheck disable=SC2086 + ${plugin}_postcheckout + + + (( RESULT = RESULT + $? )) + if [[ ${RESULT} != 0 ]] ; then + output_to_console 1 + output_to_jira 1 + cleanup_and_exit 1 + fi + fi + done +} + +## @description Driver to execute _preapply routines +## @audience private +## @stability evolving +## @replaceable no +function preapply +{ + local routine + local plugin + + for routine in precheck_without_patch check_author \ + check_modified_unittests + do + + hadoop_debug "Running ${routine}" + ${routine} + + (( RESULT = RESULT + $? )) + done + + for plugin in ${PLUGINS}; do + if declare -f ${plugin}_preapply >/dev/null 2>&1; then + + hadoop_debug "Running ${plugin}_preapply" + #shellcheck disable=SC2086 + ${plugin}_preapply + + (( RESULT = RESULT + $? )) + fi + done +} + +## @description Driver to execute _postapply routines +## @audience private +## @stability evolving +## @replaceable no +function postapply +{ + local routine + local plugin + local retval + + check_javac + retval=$? + if [[ ${retval} -gt 1 ]] ; then + output_to_console 1 + output_to_jira 1 + cleanup_and_exit 1 + fi + + ((RESULT = RESULT + retval)) + + for routine in check_javadoc check_apachelicense check_site + do + + hadoop_debug "Running ${routine}" + $routine + + (( RESULT = RESULT + $? )) + + done + + for plugin in ${PLUGINS}; do + if declare -f ${plugin}_postapply >/dev/null 2>&1; then + + hadoop_debug "Running ${plugin}_postapply" + #shellcheck disable=SC2086 + ${plugin}_postapply + (( RESULT = RESULT + $? )) + fi + done +} + +## @description Driver to execute _postinstall routines +## @audience private +## @stability evolving +## @replaceable no +function postinstall +{ + local routine + local plugin + + for routine in check_mvn_eclipse check_findbugs + do + hadoop_debug "Running ${routine}" + ${routine} + (( RESULT = RESULT + $? )) + done + + for plugin in ${PLUGINS}; do + if declare -f ${plugin}_postinstall >/dev/null 2>&1; then + hadoop_debug "Running ${plugin}_postinstall" + #shellcheck disable=SC2086 + ${plugin}_postinstall + (( RESULT = RESULT + $? )) + fi + done + +} + +## @description Driver to execute _tests routines +## @audience private +## @stability evolving +## @replaceable no +function runtests +{ + local plugin + + ### Run tests for Jenkins or if explictly asked for by a developer + if [[ ${JENKINS} == "true" || ${RUN_TESTS} == "true" ]] ; then + + check_unittests + + (( RESULT = RESULT + $? )) + fi + + for plugin in ${PLUGINS}; do + if declare -f ${plugin}_tests >/dev/null 2>&1; then + hadoop_debug "Running ${plugin}_tests" + #shellcheck disable=SC2086 + ${plugin}_tests + (( RESULT = RESULT + $? )) + fi + done +} + +## @description Import content from test-patch.d +## @audience private +## @stability evolving +## @replaceable no +function importplugins +{ + local i + local files + + if [[ -d "${BINDIR}/test-patch.d" ]]; then + files=(${BINDIR}/test-patch.d/*.sh) + fi + + for i in "${files[@]}"; do + hadoop_debug "Importing ${i}" + . "${i}" + done +} + +## @description Register test-patch.d plugins +## @audience public +## @stability stable +## @replaceable no +function add_plugin +{ + PLUGINS="${PLUGINS} $1" } ############################################################################### ############################################################################### ############################################################################### -JIRA_COMMENT="" -JIRA_COMMENT_FOOTER="Console output: $BUILD_URL/console +big_console_header "Bootstrapping test harness" -This message is automatically generated." +setup_defaults -### Check if arguments to the script have been specified properly or not -parseArgs $@ -cd $BASEDIR +parse_args "$@" -checkout +importplugins + +locate_patch + +find_changed_files + +determine_needed_tests + +git_checkout RESULT=$? -if [[ $JENKINS == "true" ]] ; then - if [[ $RESULT != 0 ]] ; then +if [[ ${JENKINS} == "true" ]] ; then + if [[ ${RESULT} != 0 ]] ; then exit 100 fi fi -downloadPatch -verifyPatch -(( RESULT = RESULT + $? )) -if [[ $RESULT != 0 ]] ; then - submitJiraComment 1 - cleanupAndExit 1 -fi -prebuildWithoutPatch -(( RESULT = RESULT + $? )) -if [[ $RESULT != 0 ]] ; then - submitJiraComment 1 - cleanupAndExit 1 -fi -checkAuthor -(( RESULT = RESULT + $? )) -if [[ $JENKINS == "true" ]] ; then - cleanUpXml -fi -checkTests -(( RESULT = RESULT + $? )) -applyPatch -APPLY_PATCH_RET=$? -(( RESULT = RESULT + $APPLY_PATCH_RET )) -if [[ $APPLY_PATCH_RET != 0 ]] ; then - submitJiraComment 1 - cleanupAndExit 1 -fi -checkJavacWarnings -JAVAC_RET=$? -#2 is returned if the code could not compile -if [[ $JAVAC_RET == 2 ]] ; then - submitJiraComment 1 - cleanupAndExit 1 -fi -(( RESULT = RESULT + $JAVAC_RET )) -checkJavadocWarnings -(( RESULT = RESULT + $? )) -### Checkstyle not implemented yet -#checkStyle -#(( RESULT = RESULT + $? )) -buildAndInstall -checkEclipseGeneration -(( RESULT = RESULT + $? )) -checkFindbugsWarnings -(( RESULT = RESULT + $? )) -checkReleaseAuditWarnings -(( RESULT = RESULT + $? )) -### Run tests for Jenkins or if explictly asked for by a developer -if [[ $JENKINS == "true" || $RUN_TESTS == "true" ]] ; then - runTests - (( RESULT = RESULT + $? )) -fi -JIRA_COMMENT_FOOTER="Test results: $BUILD_URL/testReport/ -$JIRA_COMMENT_FOOTER" +check_reexec -submitJiraComment $RESULT -cleanupAndExit $RESULT +postcheckout + +find_changed_modules + +preapply + +apply_patch_file + +postapply + +check_mvn_install + +postinstall + +runtests + +close_jira_table + +output_to_console ${RESULT} +output_to_jira ${RESULT} +cleanup_and_exit ${RESULT} diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 5c6d44ae1d..bcbffb73c1 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -447,6 +447,8 @@ Release 2.8.0 - UNRELEASED INCOMPATIBLE CHANGES + HADOOP-11746. rewrite test-patch.sh (aw) + NEW FEATURES HADOOP-11226. Add a configuration to set ipc.Client's traffic class with