diff --git a/hadoop-ozone/dev-support/checks/README.md b/hadoop-ozone/dev-support/checks/README.md new file mode 100755 index 0000000000..ba7202cab1 --- /dev/null +++ b/hadoop-ozone/dev-support/checks/README.md @@ -0,0 +1,27 @@ + + +# Ozone checks + +This directory contains a collection of easy-to-use helper scripts to execute various type of tests on the ozone/hdds codebase. + +The contract of the scripts are very simple: + + 1. Executing the scripts without any parameter will check the hdds/ozone project + 2. Shell exit code represents the result of the check (if failed, exits with non-zero code) + 3. Detailed information may be saved to the $OUTPUT_DIR (if it's not set, root level ./target will be used). + 4. The standard output should contain all the log about the build AND the results. + 5. The content of the $OUTPUT_DIR can be: + * `summary.html`/`summary.md`/`summary.txt`: contains a human readable overview about the failed tests (used by reporting) + * `failures`: contains a simple number (used by reporting) diff --git a/hadoop-ozone/dev-support/checks/_mvn_unit_report.sh b/hadoop-ozone/dev-support/checks/_mvn_unit_report.sh new file mode 100755 index 0000000000..bb29d40fa6 --- /dev/null +++ b/hadoop-ozone/dev-support/checks/_mvn_unit_report.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +REPORT_DIR=${REPORT_DIR:-$PWD} + +## generate summary txt file +find "." -name 'TEST*.xml' -print0 \ + | xargs -n1 -0 "grep" -l -E "> "${REPORT_DIR}/summary.txt" + +#Collect of all of the report failes of FAILED tests +while IFS= read -r -d '' dir; do + while IFS=$'\n' read -r file; do + DIR_OF_TESTFILE=$(dirname "$file") + NAME_OF_TESTFILE=$(basename "$file") + NAME_OF_TEST="${NAME_OF_TESTFILE%.*}" + DESTDIRNAME=$(realpath --relative-to="$PWD" "$DIR_OF_TESTFILE/../..") + mkdir -p "$REPORT_DIR/$DESTDIRNAME" + #shellcheck disable=SC2086 + cp -r "$DIR_OF_TESTFILE"/*$NAME_OF_TEST* "$REPORT_DIR/$DESTDIRNAME/" + done < <(grep -l -r FAILURE --include="*.txt" "$dir" | grep -v output.txt) +done < <(find "." -name surefire-reports -print0) + +## generate summary markdown file +export SUMMARY_FILE="$REPORT_DIR/summary.md" +for TEST_RESULT_FILE in $(find "$REPORT_DIR" -name "*.txt" | grep -v output); do + + FAILURES=$(grep FAILURE "$TEST_RESULT_FILE" | grep "Tests run" | awk '{print $18}' | sort | uniq) + + for FAILURE in $FAILURES; do + TEST_RESULT_LOCATION="$(realpath --relative-to="$REPORT_DIR" "$TEST_RESULT_FILE")" + TEST_OUTPUT_LOCATION="${TEST_RESULT_LOCATION//.txt/-output.txt/}" + printf " * [%s](%s) ([output](%s))\n" "$FAILURE" "$TEST_RESULT_LOCATION" "$TEST_OUTPUT_LOCATION" >> "$SUMMARY_FILE" + done +done + +if [ -s "$SUMMARY_FILE" ]; then + printf "# Failing tests: \n\n" | cat - "$SUMMARY_FILE" > temp && mv temp "$SUMMARY_FILE" +fi + +## generate counter +wc -l "$REPORT_DIR/summary.txt" | awk '{print $1}'> "$REPORT_DIR/failures" diff --git a/hadoop-ozone/dev-support/checks/acceptance.sh b/hadoop-ozone/dev-support/checks/acceptance.sh index 1e80ad4e48..ee03c587d4 100755 --- a/hadoop-ozone/dev-support/checks/acceptance.sh +++ b/hadoop-ozone/dev-support/checks/acceptance.sh @@ -16,7 +16,20 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" cd "$DIR/../../.." || exit 1 +REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/acceptance"} +mkdir -p "$REPORT_DIR" + OZONE_VERSION=$(grep "" "$DIR/../../pom.xml" | sed 's/<[^>]*>//g'| sed 's/^[ \t]*//') -cd "$DIR/../../dist/target/ozone-$OZONE_VERSION/compose" || exit 1 +DIST_DIR="$DIR/../../dist/target/ozone-$OZONE_VERSION" + +if [ ! -d "$DIST_DIR" ]; then + echo "Distribution dir is missing. Doing a full build" + "$DIR/build.sh" +fi + +cd "$DIST_DIR/compose" || exit 1 ./test-all.sh -exit $? +RES=$? +cp result/* "$REPORT_DIR/" +cp "$REPORT_DIR/log.html" "$REPORT_DIR/summary.html" +exit $RES diff --git a/hadoop-ozone/dev-support/checks/author.sh b/hadoop-ozone/dev-support/checks/author.sh index f50a39627b..92903f92ea 100755 --- a/hadoop-ozone/dev-support/checks/author.sh +++ b/hadoop-ozone/dev-support/checks/author.sh @@ -16,12 +16,18 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" cd "$DIR/../../.." || exit 1 -#hide this tring to not confuse yetus +REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/author"} +mkdir -p "$REPORT_DIR" +REPORT_FILE="$REPORT_DIR/summary.txt" + +#hide this string to not confuse yetus AUTHOR="uthor" AUTHOR="@a${AUTHOR}" -if grep -r --include="*.java" "$AUTHOR" .; then - exit 1 -else - exit 0 +grep -r --include="*.java" "$AUTHOR" . | tee "$REPORT_FILE" + +wc -l "$REPORT_FILE" | awk '{print $1}'> "$REPORT_DIR/failures" + +if [[ -s "${REPORT_FILE}" ]]; then + exit 1 fi diff --git a/hadoop-ozone/dev-support/checks/checkstyle.sh b/hadoop-ozone/dev-support/checks/checkstyle.sh index c4de528823..ee11e389ee 100755 --- a/hadoop-ozone/dev-support/checks/checkstyle.sh +++ b/hadoop-ozone/dev-support/checks/checkstyle.sh @@ -16,14 +16,18 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" cd "$DIR/../../.." || exit 1 +REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/checkstyle"} +mkdir -p "$REPORT_DIR" +REPORT_FILE="$REPORT_DIR/summary.txt" + mvn -B -fn checkstyle:check -f pom.ozone.xml #Print out the exact violations with parsing XML results with sed -find "." -name checkstyle-errors.xml -print0 | xargs -0 sed '$!N; //d' +find "." -name checkstyle-errors.xml -print0 | xargs -0 sed '$!N; //d' | tee "$REPORT_FILE" -violations=$(grep -r error --include checkstyle-errors.xml .| wc -l) -if [[ $violations -gt 0 ]]; then - echo "There are $violations checkstyle violations" - exit 1 +## generate counter +wc -l "$REPORT_DIR/summary.txt" | awk '{print $1}'> "$REPORT_DIR/failures" + +if [[ -s "${REPORT_FILE}" ]]; then + exit 1 fi -exit 0 diff --git a/hadoop-ozone/dev-support/checks/findbugs.sh b/hadoop-ozone/dev-support/checks/findbugs.sh index 545ad9fd7d..3108bdd056 100755 --- a/hadoop-ozone/dev-support/checks/findbugs.sh +++ b/hadoop-ozone/dev-support/checks/findbugs.sh @@ -16,21 +16,19 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" cd "$DIR/../../.." || exit 1 -FINDBUGS_ALL_FILE=./target/findbugs-all.txt - -mkdir -p ./target -rm "$FINDBUGS_ALL_FILE" || true -touch "$FINDBUGS_ALL_FILE" - mvn -B compile -fn findbugs:check -Dfindbugs.failOnError=false -f pom.ozone.xml -find hadoop-ozone -name findbugsXml.xml -print0 | xargs -0 -n1 convertXmlToText | tee -a "${FINDBUGS_ALL_FILE}" -find hadoop-hdds -name findbugsXml.xml -print0 | xargs -0 -n1 convertXmlToText | tee -a "${FINDBUGS_ALL_FILE}" +REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/findbugs"} +mkdir -p "$REPORT_DIR" +REPORT_FILE="$REPORT_DIR/summary.txt" -bugs=$(wc -l < "$FINDBUGS_ALL_FILE") +touch "$REPORT_FILE" -if [[ ${bugs} -gt 0 ]]; then +find hadoop-ozone -name findbugsXml.xml -print0 | xargs -0 -n1 convertXmlToText | tee -a "${REPORT_FILE}" +find hadoop-hdds -name findbugsXml.xml -print0 | xargs -0 -n1 convertXmlToText | tee -a "${REPORT_FILE}" + +wc -l "$REPORT_FILE" | awk '{print $1}'> "$REPORT_DIR/failures" + +if [[ -s "${REPORT_FILE}" ]]; then exit 1 -else - exit 0 fi diff --git a/hadoop-ozone/dev-support/checks/integration.sh b/hadoop-ozone/dev-support/checks/integration.sh index 8170c2e257..ccd499d432 100755 --- a/hadoop-ozone/dev-support/checks/integration.sh +++ b/hadoop-ozone/dev-support/checks/integration.sh @@ -20,10 +20,14 @@ export MAVEN_OPTS="-Xmx4096m" mvn -B install -f pom.ozone.xml -DskipTests mvn -B -fn test -f pom.ozone.xml -pl :hadoop-ozone-integration-test,:hadoop-ozone-filesystem,:hadoop-ozone-tools \ -Dtest=\!TestMiniChaosOzoneCluster -module_failed_tests=$(find "." -name 'TEST*.xml' -print0 \ - | xargs -0 -n1 "grep" -l -E "/dev/null 2>&1 && pwd )" cd "$DIR/../../.." || exit 1 -mkdir -p target -REPORT_FILE="$DIR/../../../target/rat-aggregated.txt" -mkdir -p "$(dirname "$REPORT_FILE")" +REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/rat"} +mkdir -p "$REPORT_DIR" + +REPORT_FILE="$REPORT_DIR/summary.txt" cd hadoop-hdds || exit 1 mvn -B -fn org.apache.rat:apache-rat-plugin:0.13:check @@ -26,7 +27,11 @@ cd ../hadoop-ozone || exit 1 mvn -B -fn org.apache.rat:apache-rat-plugin:0.13:check cd "$DIR/../../.." || exit 1 + grep -r --include=rat.txt "!????" hadoop-hdds hadoop-ozone | tee "$REPORT_FILE" + +wc -l "$REPORT_FILE" | awk '{print $1}'> "$REPORT_DIR/failures" + if [[ -s "${REPORT_FILE}" ]]; then exit 1 fi diff --git a/hadoop-ozone/dev-support/checks/shellcheck.sh b/hadoop-ozone/dev-support/checks/shellcheck.sh index 637a4f863c..2b67118985 100755 --- a/hadoop-ozone/dev-support/checks/shellcheck.sh +++ b/hadoop-ozone/dev-support/checks/shellcheck.sh @@ -16,8 +16,10 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" cd "$DIR/../../.." || exit 1 -OUTPUT_FILE="$DIR/../../../target/shell-problems.txt" -mkdir -p "$(dirname "$OUTPUT_FILE")" +REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/shellcheck"} +mkdir -p "$REPORT_DIR" +REPORT_FILE="$REPORT_DIR/summary.txt" + echo "" > "$OUTPUT_FILE" if [[ "$(uname -s)" = "Darwin" ]]; then find hadoop-hdds hadoop-ozone -type f -perm '-500' @@ -26,8 +28,10 @@ else fi \ | grep -v -e target/ -e node_modules/ -e '\.\(ico\|py\|yml\)$' \ | xargs -n1 shellcheck \ - | tee "$OUTPUT_FILE" + | tee "$REPORT_FILE" -if [ "$(cat "$OUTPUT_FILE")" ]; then +wc -l "$REPORT_FILE" | awk '{print $1}'> "$REPORT_DIR/failures" + +if [[ -s "${REPORT_FILE}" ]]; then exit 1 fi diff --git a/hadoop-ozone/dev-support/checks/unit.sh b/hadoop-ozone/dev-support/checks/unit.sh index 473f7147f0..9429026bd7 100755 --- a/hadoop-ozone/dev-support/checks/unit.sh +++ b/hadoop-ozone/dev-support/checks/unit.sh @@ -13,12 +13,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd "$DIR/../../.." || exit 1 + export MAVEN_OPTS="-Xmx4096m" -mvn -fn test -f pom.ozone.xml -pl \!:hadoop-ozone-integration-test,\!:hadoop-ozone-filesystem,\!:hadoop-ozone-tools -module_failed_tests=$(find "." -name 'TEST*.xml' -print0 \ - | xargs -n1 -0 "grep" -l -E "