HDDS-2030. Generate simplifed reports by the dev-support/checks/*.sh scripts
Signed-off-by: Anu Engineer <aengineer@apache.org> Co-Authored-By: Doroszlai, Attila <6454655+adoroszlai@users.noreply.github.com>
This commit is contained in:
Márton Elek
Anu Engineer
parent
56f042c48f
commit
c8d61ffef6
27
hadoop-ozone/dev-support/checks/README.md
Executable file
27
hadoop-ozone/dev-support/checks/README.md
Executable file
@ -0,0 +1,27 @@
|
||||
<!---
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. See accompanying LICENSE file.
|
||||
-->
|
||||
|
||||
# Ozone checks
|
||||
|
||||
This directory contains a collection of easy-to-use helper scripts to execute various type of tests on the ozone/hdds codebase.
|
||||
|
||||
The contract of the scripts are very simple:
|
||||
|
||||
1. Executing the scripts without any parameter will check the hdds/ozone project
|
||||
2. Shell exit code represents the result of the check (if failed, exits with non-zero code)
|
||||
3. Detailed information may be saved to the $OUTPUT_DIR (if it's not set, root level ./target will be used).
|
||||
4. The standard output should contain all the log about the build AND the results.
|
||||
5. The content of the $OUTPUT_DIR can be:
|
||||
* `summary.html`/`summary.md`/`summary.txt`: contains a human readable overview about the failed tests (used by reporting)
|
||||
* `failures`: contains a simple number (used by reporting)
|
||||
66
hadoop-ozone/dev-support/checks/_mvn_unit_report.sh
Executable file
66
hadoop-ozone/dev-support/checks/_mvn_unit_report.sh
Executable file
@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env bash
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
REPORT_DIR=${REPORT_DIR:-$PWD}
|
||||
|
||||
## generate summary txt file
|
||||
find "." -name 'TEST*.xml' -print0 \
|
||||
| xargs -n1 -0 "grep" -l -E "<failure|<error" \
|
||||
| awk -F/ '{sub("'"TEST-"'",""); sub(".xml",""); print $NF}' \
|
||||
| tee "$REPORT_DIR/summary.txt"
|
||||
|
||||
#Copy heap dump and dump leftovers
|
||||
find "." -name "*.hprof" -exec cp {} "$REPORT_DIR/" \;
|
||||
find "." -name "*.dump" -exec cp {} "$REPORT_DIR/" \;
|
||||
|
||||
## Add the tests where the JVM is crashed
|
||||
grep -A1 'Crashed tests' "${REPORT_DIR}/output.log" \
|
||||
| grep -v -e 'Crashed tests' -e '--' \
|
||||
| cut -f2- -d' ' \
|
||||
| sort -u >> "${REPORT_DIR}/summary.txt"
|
||||
|
||||
#Collect of all of the report failes of FAILED tests
|
||||
while IFS= read -r -d '' dir; do
|
||||
while IFS=$'\n' read -r file; do
|
||||
DIR_OF_TESTFILE=$(dirname "$file")
|
||||
NAME_OF_TESTFILE=$(basename "$file")
|
||||
NAME_OF_TEST="${NAME_OF_TESTFILE%.*}"
|
||||
DESTDIRNAME=$(realpath --relative-to="$PWD" "$DIR_OF_TESTFILE/../..")
|
||||
mkdir -p "$REPORT_DIR/$DESTDIRNAME"
|
||||
#shellcheck disable=SC2086
|
||||
cp -r "$DIR_OF_TESTFILE"/*$NAME_OF_TEST* "$REPORT_DIR/$DESTDIRNAME/"
|
||||
done < <(grep -l -r FAILURE --include="*.txt" "$dir" | grep -v output.txt)
|
||||
done < <(find "." -name surefire-reports -print0)
|
||||
|
||||
## generate summary markdown file
|
||||
export SUMMARY_FILE="$REPORT_DIR/summary.md"
|
||||
for TEST_RESULT_FILE in $(find "$REPORT_DIR" -name "*.txt" | grep -v output); do
|
||||
|
||||
FAILURES=$(grep FAILURE "$TEST_RESULT_FILE" | grep "Tests run" | awk '{print $18}' | sort | uniq)
|
||||
|
||||
for FAILURE in $FAILURES; do
|
||||
TEST_RESULT_LOCATION="$(realpath --relative-to="$REPORT_DIR" "$TEST_RESULT_FILE")"
|
||||
TEST_OUTPUT_LOCATION="${TEST_RESULT_LOCATION//.txt/-output.txt/}"
|
||||
printf " * [%s](%s) ([output](%s))\n" "$FAILURE" "$TEST_RESULT_LOCATION" "$TEST_OUTPUT_LOCATION" >> "$SUMMARY_FILE"
|
||||
done
|
||||
done
|
||||
|
||||
if [ -s "$SUMMARY_FILE" ]; then
|
||||
printf "# Failing tests: \n\n" | cat - "$SUMMARY_FILE" > temp && mv temp "$SUMMARY_FILE"
|
||||
fi
|
||||
|
||||
## generate counter
|
||||
wc -l "$REPORT_DIR/summary.txt" | awk '{print $1}'> "$REPORT_DIR/failures"
|
||||
@ -16,7 +16,20 @@
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
cd "$DIR/../../.." || exit 1
|
||||
|
||||
REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/acceptance"}
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
OZONE_VERSION=$(grep "<ozone.version>" "$DIR/../../pom.xml" | sed 's/<[^>]*>//g'| sed 's/^[ \t]*//')
|
||||
cd "$DIR/../../dist/target/ozone-$OZONE_VERSION/compose" || exit 1
|
||||
DIST_DIR="$DIR/../../dist/target/ozone-$OZONE_VERSION"
|
||||
|
||||
if [ ! -d "$DIST_DIR" ]; then
|
||||
echo "Distribution dir is missing. Doing a full build"
|
||||
"$DIR/build.sh"
|
||||
fi
|
||||
|
||||
cd "$DIST_DIR/compose" || exit 1
|
||||
./test-all.sh
|
||||
exit $?
|
||||
RES=$?
|
||||
cp result/* "$REPORT_DIR/"
|
||||
cp "$REPORT_DIR/log.html" "$REPORT_DIR/summary.html"
|
||||
exit $RES
|
||||
|
||||
@ -16,12 +16,18 @@
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
cd "$DIR/../../.." || exit 1
|
||||
|
||||
#hide this tring to not confuse yetus
|
||||
REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/author"}
|
||||
mkdir -p "$REPORT_DIR"
|
||||
REPORT_FILE="$REPORT_DIR/summary.txt"
|
||||
|
||||
#hide this string to not confuse yetus
|
||||
AUTHOR="uthor"
|
||||
AUTHOR="@a${AUTHOR}"
|
||||
|
||||
if grep -r --include="*.java" "$AUTHOR" .; then
|
||||
exit 1
|
||||
else
|
||||
exit 0
|
||||
grep -r --include="*.java" "$AUTHOR" . | tee "$REPORT_FILE"
|
||||
|
||||
wc -l "$REPORT_FILE" | awk '{print $1}'> "$REPORT_DIR/failures"
|
||||
|
||||
if [[ -s "${REPORT_FILE}" ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@ -16,14 +16,18 @@
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
cd "$DIR/../../.." || exit 1
|
||||
|
||||
REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/checkstyle"}
|
||||
mkdir -p "$REPORT_DIR"
|
||||
REPORT_FILE="$REPORT_DIR/summary.txt"
|
||||
|
||||
mvn -B -fn checkstyle:check -f pom.ozone.xml
|
||||
|
||||
#Print out the exact violations with parsing XML results with sed
|
||||
find "." -name checkstyle-errors.xml -print0 | xargs -0 sed '$!N; /<file.*\n<\/file/d;P;D' | sed '/<\/.*/d;/<checkstyle.*/d;s/<error.*line="\([[:digit:]]*\)".*message="\([^"]\+\).*/ \1: \2/;s/<file name="\([^"]*\)".*/\1/;/<\?xml.*>/d'
|
||||
find "." -name checkstyle-errors.xml -print0 | xargs -0 sed '$!N; /<file.*\n<\/file/d;P;D' | sed '/<\/.*/d;/<checkstyle.*/d;s/<error.*line="\([[:digit:]]*\)".*message="\([^"]\+\).*/ \1: \2/;s/<file name="\([^"]*\)".*/\1/;/<\?xml.*>/d' | tee "$REPORT_FILE"
|
||||
|
||||
violations=$(grep -r error --include checkstyle-errors.xml .| wc -l)
|
||||
if [[ $violations -gt 0 ]]; then
|
||||
echo "There are $violations checkstyle violations"
|
||||
exit 1
|
||||
## generate counter
|
||||
wc -l "$REPORT_DIR/summary.txt" | awk '{print $1}'> "$REPORT_DIR/failures"
|
||||
|
||||
if [[ -s "${REPORT_FILE}" ]]; then
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
|
||||
@ -16,21 +16,19 @@
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
cd "$DIR/../../.." || exit 1
|
||||
|
||||
FINDBUGS_ALL_FILE=./target/findbugs-all.txt
|
||||
|
||||
mkdir -p ./target
|
||||
rm "$FINDBUGS_ALL_FILE" || true
|
||||
touch "$FINDBUGS_ALL_FILE"
|
||||
|
||||
mvn -B compile -fn findbugs:check -Dfindbugs.failOnError=false -f pom.ozone.xml
|
||||
|
||||
find hadoop-ozone -name findbugsXml.xml -print0 | xargs -0 -n1 convertXmlToText | tee -a "${FINDBUGS_ALL_FILE}"
|
||||
find hadoop-hdds -name findbugsXml.xml -print0 | xargs -0 -n1 convertXmlToText | tee -a "${FINDBUGS_ALL_FILE}"
|
||||
REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/findbugs"}
|
||||
mkdir -p "$REPORT_DIR"
|
||||
REPORT_FILE="$REPORT_DIR/summary.txt"
|
||||
|
||||
bugs=$(wc -l < "$FINDBUGS_ALL_FILE")
|
||||
touch "$REPORT_FILE"
|
||||
|
||||
if [[ ${bugs} -gt 0 ]]; then
|
||||
find hadoop-ozone -name findbugsXml.xml -print0 | xargs -0 -n1 convertXmlToText | tee -a "${REPORT_FILE}"
|
||||
find hadoop-hdds -name findbugsXml.xml -print0 | xargs -0 -n1 convertXmlToText | tee -a "${REPORT_FILE}"
|
||||
|
||||
wc -l "$REPORT_FILE" | awk '{print $1}'> "$REPORT_DIR/failures"
|
||||
|
||||
if [[ -s "${REPORT_FILE}" ]]; then
|
||||
exit 1
|
||||
else
|
||||
exit 0
|
||||
fi
|
||||
|
||||
@ -20,10 +20,14 @@ export MAVEN_OPTS="-Xmx4096m"
|
||||
mvn -B install -f pom.ozone.xml -DskipTests
|
||||
mvn -B -fn test -f pom.ozone.xml -pl :hadoop-ozone-integration-test,:hadoop-ozone-filesystem,:hadoop-ozone-tools \
|
||||
-Dtest=\!TestMiniChaosOzoneCluster
|
||||
module_failed_tests=$(find "." -name 'TEST*.xml' -print0 \
|
||||
| xargs -0 -n1 "grep" -l -E "<failure|<error"\
|
||||
| awk -F/ '{sub("'"TEST-JUNIT_TEST_OUTPUT_DIR"'",""); sub(".xml",""); print $NF}')
|
||||
if [[ -n "${module_failed_tests}" ]] ; then
|
||||
|
||||
REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/integration"}
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
# shellcheck source=hadoop-ozone/dev-support/checks/_mvn_unit_report.sh
|
||||
source "$DIR/_mvn_unit_report.sh"
|
||||
|
||||
if [[ -s "$REPORT_DIR/summary.txt" ]] ; then
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
|
||||
@ -16,9 +16,10 @@
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
cd "$DIR/../../.." || exit 1
|
||||
|
||||
mkdir -p target
|
||||
REPORT_FILE="$DIR/../../../target/rat-aggregated.txt"
|
||||
mkdir -p "$(dirname "$REPORT_FILE")"
|
||||
REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/rat"}
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
REPORT_FILE="$REPORT_DIR/summary.txt"
|
||||
|
||||
cd hadoop-hdds || exit 1
|
||||
mvn -B -fn org.apache.rat:apache-rat-plugin:0.13:check
|
||||
@ -26,7 +27,11 @@ cd ../hadoop-ozone || exit 1
|
||||
mvn -B -fn org.apache.rat:apache-rat-plugin:0.13:check
|
||||
|
||||
cd "$DIR/../../.." || exit 1
|
||||
|
||||
grep -r --include=rat.txt "!????" hadoop-hdds hadoop-ozone | tee "$REPORT_FILE"
|
||||
|
||||
wc -l "$REPORT_FILE" | awk '{print $1}'> "$REPORT_DIR/failures"
|
||||
|
||||
if [[ -s "${REPORT_FILE}" ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@ -16,8 +16,10 @@
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
cd "$DIR/../../.." || exit 1
|
||||
|
||||
OUTPUT_FILE="$DIR/../../../target/shell-problems.txt"
|
||||
mkdir -p "$(dirname "$OUTPUT_FILE")"
|
||||
REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/shellcheck"}
|
||||
mkdir -p "$REPORT_DIR"
|
||||
REPORT_FILE="$REPORT_DIR/summary.txt"
|
||||
|
||||
echo "" > "$OUTPUT_FILE"
|
||||
if [[ "$(uname -s)" = "Darwin" ]]; then
|
||||
find hadoop-hdds hadoop-ozone -type f -perm '-500'
|
||||
@ -26,8 +28,10 @@ else
|
||||
fi \
|
||||
| grep -v -e target/ -e node_modules/ -e '\.\(ico\|py\|yml\)$' \
|
||||
| xargs -n1 shellcheck \
|
||||
| tee "$OUTPUT_FILE"
|
||||
| tee "$REPORT_FILE"
|
||||
|
||||
if [ "$(cat "$OUTPUT_FILE")" ]; then
|
||||
wc -l "$REPORT_FILE" | awk '{print $1}'> "$REPORT_DIR/failures"
|
||||
|
||||
if [[ -s "${REPORT_FILE}" ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@ -13,12 +13,19 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
cd "$DIR/../../.." || exit 1
|
||||
|
||||
export MAVEN_OPTS="-Xmx4096m"
|
||||
mvn -fn test -f pom.ozone.xml -pl \!:hadoop-ozone-integration-test,\!:hadoop-ozone-filesystem,\!:hadoop-ozone-tools
|
||||
module_failed_tests=$(find "." -name 'TEST*.xml' -print0 \
|
||||
| xargs -n1 -0 "grep" -l -E "<failure|<error"\
|
||||
| awk -F/ '{sub("'"TEST-JUNIT_TEST_OUTPUT_DIR"'",""); sub(".xml",""); print $NF}')
|
||||
if [[ -n "${module_failed_tests}" ]] ; then
|
||||
mvn -B -fn test -f pom.ozone.xml -pl \!:hadoop-ozone-integration-test,\!:hadoop-ozone-filesystem,\!:hadoop-ozone-tools
|
||||
|
||||
REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/unit"}
|
||||
mkdir -p "$REPORT_DIR"
|
||||
|
||||
# shellcheck source=hadoop-ozone/dev-support/checks/_mvn_unit_report.sh
|
||||
source "$DIR/_mvn_unit_report.sh"
|
||||
|
||||
if [[ -s "$REPORT_DIR/summary.txt" ]] ; then
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
|
||||
Loading…
Reference in New Issue
Block a user