big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-12030. test-patch should only report on newly introduced findbugs warnings. (Sean Busbey via aw)

Browse Source
This commit is contained in:
Allen Wittenauer 2015-05-28 09:53:50 -07:00
parent f1cea9c6bf
commit b01d33cf86
2 changed files with 231 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

298
dev-support/test-patch.sh
View File

@ -44,6 +44,7 @@ function setup_defaults
LOAD_SYSTEM_PLUGINS=true LOAD_SYSTEM_PLUGINS=true
FINDBUGS_HOME=${FINDBUGS_HOME:-} FINDBUGS_HOME=${FINDBUGS_HOME:-}
FINDBUGS_WARNINGS_FAIL_PRECHECK=false
ECLIPSE_HOME=${ECLIPSE_HOME:-} ECLIPSE_HOME=${ECLIPSE_HOME:-}
BUILD_NATIVE=${BUILD_NATIVE:-true} BUILD_NATIVE=${BUILD_NATIVE:-true}
PATCH_BRANCH="" PATCH_BRANCH=""
@ -585,6 +586,7 @@ function hadoop_usage
echo "--debug If set, then output some extra stuff to stderr" echo "--debug If set, then output some extra stuff to stderr"
echo "--dirty-workspace Allow the local git workspace to have uncommitted changes" echo "--dirty-workspace Allow the local git workspace to have uncommitted changes"
echo "--findbugs-home=<path> Findbugs home directory (default FINDBUGS_HOME environment variable)" echo "--findbugs-home=<path> Findbugs home directory (default FINDBUGS_HOME environment variable)"
echo "--findbugs-strict-precheck If there are Findbugs warnings during precheck, fail"
echo "--issue-re=<expr> Bash regular expression to use when trying to find a jira ref in the patch name (default '^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$')" echo "--issue-re=<expr> Bash regular expression to use when trying to find a jira ref in the patch name (default '^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$')"
echo "--modulelist=<list> Specify additional modules to test (comma delimited)" echo "--modulelist=<list> Specify additional modules to test (comma delimited)"
echo "--offline Avoid connecting to the Internet" echo "--offline Avoid connecting to the Internet"
@ -666,6 +668,9 @@ function parse_args
--findbugs-home=*) --findbugs-home=*)
FINDBUGS_HOME=${i#*=} FINDBUGS_HOME=${i#*=}
;; ;;
--findbugs-strict-precheck)
FINDBUGS_WARNINGS_FAIL_PRECHECK=true
;;
--git-cmd=*) --git-cmd=*)
GIT=${i#*=} GIT=${i#*=}
;; ;;
@ -1060,6 +1065,12 @@ function precheck_without_patch
echo "Patch does not appear to need site tests." echo "Patch does not appear to need site tests."
fi fi
precheck_findbugs
if [[ $? != 0 ]] ; then
return 1
fi
add_jira_table 0 pre-patch "Pre-patch ${PATCH_BRANCH} compilation is healthy." add_jira_table 0 pre-patch "Pre-patch ${PATCH_BRANCH} compilation is healthy."
return 0 return 0
} }
@ -1838,40 +1849,84 @@ function check_mvn_install
return ${retval} return ${retval}
} }
## @description Verify patch does not trigger any findbugs warnings ## @description are the needed bits for findbugs present?
## @audience private
## @stability evolving
## @replaceable no
## @return 0 findbugs will work for our use
## @return 1 findbugs is missing some component
function findbugs_is_installed
{
if [[ ! -e "${FINDBUGS_HOME}/bin/findbugs" ]]; then
printf "\n\n%s is not executable.\n\n" "${FINDBUGS_HOME}/bin/findbugs"
add_jira_table -1 findbugs "Findbugs is not installed."
return 1
fi
return 0
}
## @description Run the maven findbugs plugin and record found issues in a bug database
## @audience private ## @audience private
## @stability evolving ## @stability evolving
## @replaceable no ## @replaceable no
## @return 0 on success ## @return 0 on success
## @return 1 on failure ## @return 1 on failure
function check_findbugs function findbugs_mvnrunner
{ {
local findbugs_version local name=$1
local modules=${CHANGED_MODULES} local logfile=$2
local rc=0 local warnings_file=$3
echo_and_redirect "${logfile}" "${MVN}" clean test findbugs:findbugs -DskipTests \
"-D${PROJECT_NAME}PatchProcess" < /dev/null
if [[ $? != 0 ]]; then
return 1
fi
cp target/findbugsXml.xml "${warnings_file}.xml"
"${FINDBUGS_HOME}/bin/setBugDatabaseInfo" -name "${name}" \
"${warnings_file}.xml" "${warnings_file}.xml"
if [[ $? != 0 ]]; then
return 1
fi
"${FINDBUGS_HOME}/bin/convertXmlToText" -html "${warnings_file}.xml" \
"${warnings_file}.html"
if [[ $? != 0 ]]; then
return 1
fi
return 0
}
## @description Track pre-existing findbugs warnings
## @audience private
## @stability evolving
## @replaceable no
## @return 0 on success
## @return 1 on failure
function precheck_findbugs
{
local -r mypwd=$(pwd)
local module_suffix local module_suffix
local findbugsWarnings=0 local modules=${CHANGED_MODULES}
local relative_file local module
local newFindbugsWarnings local findbugs_version
local findbugsWarnings local rc=0
local line local module_findbugs_warnings
local firstpart local findbugs_warnings=0
local secondpart
big_console_header "Determining number of patched Findbugs warnings."
verify_needed_test findbugs verify_needed_test findbugs
if [[ $? == 0 ]]; then if [[ $? == 0 ]]; then
echo "Patch does not touch any java files. Skipping findbugs." echo "Patch does not appear to need findbugs tests."
return 0 return 0
fi fi
start_clock echo "findbugs baseline for ${mypwd}"
if [[ ! -e "${FINDBUGS_HOME}/bin/findbugs" ]]; then findbugs_is_installed
printf "\n\n%s is not executable.\n\n" "${FINDBUGS_HOME}/bin/findbugs" if [[ $? != 0 ]]; then
add_jira_table -1 findbugs "Findbugs is not installed."
return 1 return 1
fi fi
@ -1880,67 +1935,170 @@ function check_findbugs
pushd "${module}" >/dev/null pushd "${module}" >/dev/null
echo " Running findbugs in ${module}" echo " Running findbugs in ${module}"
module_suffix=$(basename "${module}") module_suffix=$(basename "${module}")
echo_and_redirect "${PATCH_DIR}/patchFindBugsOutput${module_suffix}.txt" "${MVN}" clean test findbugs:findbugs -DskipTests -D${PROJECT_NAME}PatchProcess \ findbugs_mvnrunner "${PATCH_BRANCH}" \
< /dev/null "${PATCH_DIR}/${PATCH_BRANCH}FindBugsOutput${module_suffix}.txt" \
"${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}"
(( rc = rc + $? )) (( rc = rc + $? ))
if [[ "${FINDBUGS_WARNINGS_FAIL_PRECHECK}" == "true" ]]; then
#shellcheck disable=SC2016
module_findbugs_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first \
"${PATCH_BRANCH}" \
"${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}".xml \
"${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}".xml \
| ${AWK} '{print $1}')
if [[ $? != 0 ]]; then
popd >/dev/null
return 1
fi
findbugs_warnings=$((findbugs_warnings+module_findbugs_warnings))
if [[ ${module_findbugs_warnings} -gt 0 ]] ; then
add_jira_footer "Pre-patch Findbugs warnings" "@@BASE@@/${PATCH_BRANCH}FindbugsWarnings${module_suffix}.html"
fi
fi
popd >/dev/null
done
#shellcheck disable=SC2016
findbugs_version=$(${AWK} 'match($0, /findbugs-maven-plugin:[^:]*:findbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/${PATCH_BRANCH}FindBugsOutput${module_suffix}.txt")
if [[ ${rc} -ne 0 ]]; then
echo "Pre-patch ${PATCH_BRANCH} findbugs is broken?"
add_jira_table -1 pre-patch "Findbugs (version ${findbugs_version}) appears to be broken on ${PATCH_BRANCH}."
return 1
fi
if [[ "${FINDBUGS_WARNINGS_FAIL_PRECHECK}" == "true" && \
${findbugs_warnings} -gt 0 ]] ; then
echo "Pre-patch ${PATCH_BRANCH} findbugs has ${findbugs_warnings} warnings."
add_jira_table -1 pre-patch "Pre-patch ${PATCH_BRANCH} has ${findbugs_warnings} extant Findbugs (version ${findbugs_version}) warnings."
return 1
fi
return 0
}
## @description Verify patch does not trigger any findbugs warnings
## @audience private
## @stability evolving
## @replaceable no
## @return 0 on success
## @return 1 on failure
function check_findbugs
{
local rc=0
local module
local modules=${CHANGED_MODULES}
local module_suffix
local combined_xml
local newBugs
local new_findbugs_warnings
local new_findbugs_fixed_warnings
local findbugs_warnings=0
local findbugs_fixed_warnings=0
local line
local firstpart
local secondpart
local findbugs_version
verify_needed_test findbugs
if [[ $? == 0 ]]; then
return 0
fi
big_console_header "Determining number of patched Findbugs warnings."
start_clock
findbugs_is_installed
if [[ $? != 0 ]]; then
return 1
fi
for module in ${modules}
do
pushd "${module}" >/dev/null
echo " Running findbugs in ${module}"
module_suffix=$(basename "${module}")
findbugs_mvnrunner patch \
"${PATCH_DIR}/patchFindBugsOutput${module_suffix}.txt" \
"${PATCH_DIR}/patchFindbugsWarnings${module_suffix}"
if [[ $? != 0 ]] ; then
((rc = rc +1))
echo "Post-patch findbugs compilation is broken."
add_jira_table -1 findbugs "Post-patch findbugs ${module} compilation is broken."
continue
fi
combined_xml="$PATCH_DIR/combinedFindbugsWarnings${module_suffix}.xml"
newBugs="${PATCH_DIR}/newPatchFindbugsWarnings${module_suffix}"
"${FINDBUGS_HOME}/bin/computeBugHistory" -useAnalysisTimes -withMessages \
-output "${combined_xml}" \
"${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}.xml" \
"${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml"
if [[ $? != 0 ]]; then
popd >/dev/null
return 1
fi
#shellcheck disable=SC2016
new_findbugs_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first patch \
"${combined_xml}" "${newBugs}.xml" | ${AWK} '{print $1}')
if [[ $? != 0 ]]; then
popd >/dev/null
return 1
fi
#shellcheck disable=SC2016
new_findbugs_fixed_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -fixed patch \
"${combined_xml}" "${newBugs}.xml" | ${AWK} '{print $1}')
if [[ $? != 0 ]]; then
popd >/dev/null
return 1
fi
echo "Found ${new_findbugs_warnings} new Findbugs warnings and ${new_findbugs_fixed_warnings} newly fixed warnings."
findbugs_warnings=$((findbugs_warnings+new_findbugs_warnings))
findbugs_fixed_warnings=$((findbugs_fixed_warnings+new_findbugs_fixed_warnings))
"${FINDBUGS_HOME}/bin/convertXmlToText" -html "${newBugs}.xml" \
"${newBugs}.html"
if [[ $? != 0 ]]; then
popd >/dev/null
return 1
fi
if [[ ${new_findbugs_warnings} -gt 0 ]] ; then
populate_test_table FindBugs "module:${module_suffix}"
while read line; do
firstpart=$(echo "${line}" | cut -f2 -d:)
secondpart=$(echo "${line}" | cut -f9- -d' ')
add_jira_test_table "" "${firstpart}:${secondpart}"
done < <("${FINDBUGS_HOME}/bin/convertXmlToText" "${newBugs}.xml")
add_jira_footer "Findbugs warnings" "@@BASE@@/newPatchFindbugsWarnings${module_suffix}.html"
fi
popd >/dev/null popd >/dev/null
done done
#shellcheck disable=SC2016 #shellcheck disable=SC2016
findbugs_version=$(${AWK} 'match($0, /findbugs-maven-plugin:[^:]*:findbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/patchFindBugsOutput${module_suffix}.txt") findbugs_version=$(${AWK} 'match($0, /findbugs-maven-plugin:[^:]*:findbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/patchFindBugsOutput${module_suffix}.txt")
if [[ ${rc} -ne 0 ]]; then if [[ ${findbugs_warnings} -gt 0 ]] ; then
add_jira_table -1 findbugs "The patch appears to cause Findbugs (version ${findbugs_version}) to fail." add_jira_table -1 findbugs "The patch appears to introduce ${findbugs_warnings} new Findbugs (version ${findbugs_version}) warnings."
return 1 return 1
fi fi
while read file if [[ ${findbugs_fixed_warnings} -gt 0 ]] ; then
do add_jira_table +1 findbugs "The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings, and fixes ${findbugs_fixed_warnings} pre-existing warnings."
relative_file=${file#${BASEDIR}/} # strip leading ${BASEDIR} prefix else
if [[ ${relative_file} != "target/findbugsXml.xml" ]]; then add_jira_table +1 findbugs "The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings."
module_suffix=${relative_file%/target/findbugsXml.xml} # strip trailing path
module_suffix=$(basename "${module_suffix}")
fi
cp "${file}" "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml"
"${FINDBUGS_HOME}/bin/setBugDatabaseInfo" -timestamp "01/01/2000" \
"${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml" \
"${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml"
#shellcheck disable=SC2016
newFindbugsWarnings=$("${FINDBUGS_HOME}/bin/filterBugs" \
-first "01/01/2000" "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml" \
"${PATCH_DIR}/newPatchFindbugsWarnings${module_suffix}.xml" \
| ${AWK} '{print $1}')
echo "Found $newFindbugsWarnings Findbugs warnings ($file)"
findbugsWarnings=$((findbugsWarnings+newFindbugsWarnings))
"${FINDBUGS_HOME}/bin/convertXmlToText" -html \
"${PATCH_DIR}/newPatchFindbugsWarnings${module_suffix}.xml" \
"${PATCH_DIR}/newPatchFindbugsWarnings${module_suffix}.html"
if [[ ${newFindbugsWarnings} -gt 0 ]] ; then
populate_test_table FindBugs "module:${module_suffix}"
while read line; do
firstpart=$(echo "${line}" | cut -f2 -d:)
secondpart=$(echo "${line}" | cut -f9- -d' ')
add_jira_test_table "" "${firstpart}:${secondpart}"
done < <("${FINDBUGS_HOME}/bin/convertXmlToText" \
"${PATCH_DIR}/newPatchFindbugsWarnings${module_suffix}.xml")
add_jira_footer "Findbugs warnings" "@@BASE@@/newPatchFindbugsWarnings${module_suffix}.html"
fi
done < <(find "${BASEDIR}" -name findbugsXml.xml)
if [[ ${findbugsWarnings} -gt 0 ]] ; then
add_jira_table -1 findbugs "The patch appears to introduce ${findbugsWarnings} new Findbugs (version ${findbugs_version}) warnings."
return 1
fi fi
add_jira_table +1 findbugs "The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings."
return 0 return 0
} }

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -606,6 +606,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11594. Improve the readability of site index of documentation. HADOOP-11594. Improve the readability of site index of documentation.
(Masatake Iwasaki via aajisaka) (Masatake Iwasaki via aajisaka)
HADOOP-12030. test-patch should only report on newly introduced
findbugs warnings. (Sean Busbey via aw)
OPTIMIZATIONS OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp HADOOP-11785. Reduce the number of listStatus operation in distcp