big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDDS-1735. Create separate unit and integration test executor dev-support script. Contributed by Elek, Marton.

Browse Source
This commit is contained in:
Anu Engineer 2019-07-12 09:25:00 -07:00 committed by GitHub
commit 4a70a0d816
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 112 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-ozone/dev-support/checks/acceptance.sh
View File

@ -14,6 +14,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1
export HADOOP_VERSION=3
"$DIR/../../../hadoop-ozone/dist/target/ozone-*-SNAPSHOT/compose/test-all.sh"
OZONE_VERSION=$(grep "<ozone.version>" "$DIR/../../pom.xml" | sed 's/<[^>]*>//g'| sed 's/^[ \t]*//')
cd "$DIR/../../dist/target/ozone-$OZONE_VERSION/compose" || exit 1
./test-all.sh
exit $?

14
hadoop-ozone/dev-support/checks/author.sh
View File

@ -13,10 +13,16 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
mkdir -p ./target
grep -r --include="*.java" "@author" .
if [ $? -gt 0 ]; then
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1
#hide this tring to not confuse yetus
AUTHOR="uthor"
AUTHOR="@a${AUTHOR}"
grep -r --include="*.java" "$AUTHOR" .
if grep -r --include="*.java" "$AUTHOR" .; then
exit 0
else
exit -1
exit 1
fi

5
hadoop-ozone/dev-support/checks/build.sh
View File

@ -13,6 +13,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1
export MAVEN_OPTS="-Xmx4096m"
mvn -am -pl :hadoop-ozone-dist -P hdds -Dmaven.javadoc.skip=true -DskipTests clean install
mvn -B -f pom.ozone.xml -Dmaven.javadoc.skip=true -DskipTests clean install
exit $?

10
hadoop-ozone/dev-support/checks/checkstyle.sh
View File

@ -13,11 +13,17 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
mvn -fn checkstyle:check -am -pl :hadoop-ozone-dist -Phdds
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1
mvn -B -fn checkstyle:check -f pom.ozone.xml
#Print out the exact violations with parsing XML results with sed
find "." -name checkstyle-errors.xml -print0 | xargs -0 sed '$!N; /<file.*\n<\/file/d;P;D' | sed '/<\/.*/d;/<checkstyle.*/d;s/<error.*line="\([[:digit:]]*\)".*message="\([^"]\+\).*/ \1: \2/;s/<file name="\([^"]*\)".*/\1/;/<\?xml.*>/d'
violations=$(grep -r error --include checkstyle-errors.xml .| wc -l)
if [[ $violations -gt 0 ]]; then
echo "There are $violations checkstyle violations"
exit -1
exit 1
fi
exit 0

12
hadoop-ozone/dev-support/checks/findbugs.sh
View File

@ -13,6 +13,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1
FINDBUGS_ALL_FILE=./target/findbugs-all.txt
@ -20,15 +22,15 @@ mkdir -p ./target
rm "$FINDBUGS_ALL_FILE" || true
touch "$FINDBUGS_ALL_FILE"
mvn -fn findbugs:check -Dfindbugs.failOnError=false -am -pl :hadoop-ozone-dist -Phdds
mvn -B compile -fn findbugs:check -Dfindbugs.failOnError=false -f pom.ozone.xml
find hadoop-ozone -name findbugsXml.xml | xargs -n1 convertXmlToText | tee -a "${FINDBUGS_ALL_FILE}"
find hadoop-hdds -name findbugsXml.xml | xargs -n1 convertXmlToText | tee -a "${FINDBUGS_ALL_FILE}"
find hadoop-ozone -name findbugsXml.xml -print0 | xargs -0 -n1 convertXmlToText | tee -a "${FINDBUGS_ALL_FILE}"
find hadoop-hdds -name findbugsXml.xml -print0 | xargs -0 -n1 convertXmlToText | tee -a "${FINDBUGS_ALL_FILE}"
bugs=$(cat "$FINDBUGS_ALL_FILE" | wc -l)
bugs=$(wc -l < "$FINDBUGS_ALL_FILE")
if [[ ${bugs} -gt 0 ]]; then
exit -1
exit 1
else
exit 0
fi

28
hadoop-ozone/dev-support/checks/integration.sh Executable file
View File

@ -0,0 +1,28 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1
export MAVEN_OPTS="-Xmx4096m"
mvn -B install -f pom.ozone.xml -DskipTests
mvn -B -fn test -f pom.ozone.xml -pl :hadoop-ozone-integration-test,:hadoop-ozone-filesystem
module_failed_tests=$(find "." -name 'TEST*.xml' -print0 \
| xargs -0 -n1 "grep" -l -E "<failure|<error"\
| awk -F/ '{sub("'"TEST-JUNIT_TEST_OUTPUT_DIR"'",""); sub(".xml",""); print $NF}')
if [[ -n "${module_failed_tests}" ]] ; then
exit 1
fi
exit 0

7
hadoop-ozone/dev-support/checks/isolation.sh
View File

@ -13,12 +13,15 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
hadooplines=$(git diff --name-only HEAD~1..HEAD | grep -v hadoop-ozone | grep -v hadoop-hdds | wc -l )
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1
hadooplines=$(git diff --name-only HEAD~1..HEAD | grep -v hadoop-ozone | grep -c -v hadoop-hdds )
if [ "$hadooplines" == "0" ]; then
echo "Only ozone/hdds subprojects are changed"
exit 0
else
echo "Main hadoop projects are changed in an ozone patch."
echo "Please do it in a HADOOP/HDFS patch and test it with hadoop precommit tests"
exit -1
exit 1
fi

17
hadoop-ozone/dev-support/checks/rat.sh
View File

@ -13,12 +13,21 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1
mkdir -p target
rm target/rat-aggregated.txt
mvn -fn org.apache.rat:apache-rat-plugin:0.13:check -am -pl :hadoop-ozone-dist -Phdds
grep -r --include=rat.txt "!????" | tee ./target/rat-aggregated.txt
REPORT_FILE="$DIR/../../../target/rat-aggretaged.txt"
mkdir -p "$(dirname "$REPORT_FILE")"
cd hadoop-hdds || exit 1
mvn -B -fn org.apache.rat:apache-rat-plugin:0.13:check
cd ../hadoop-ozone || exit 1
mvn -B -fn org.apache.rat:apache-rat-plugin:0.13:check
cd "$DIR/../../.." || exit 1
grep -r --include=rat.txt "!????" | tee "$REPORT_FILE"
if [ "$(cat target/rat-aggregated.txt)" ]; then
exit -1
exit 1
fi

28
hadoop-ozone/dev-support/checks/shellcheck.sh Executable file
View File

@ -0,0 +1,28 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1
OUTPUT_FILE="$DIR/../../../target/shell-problems.txt"
mkdir -p "$(dirname "$OUTPUT_FILE")"
echo "" > "$OUTPUT_FILE"
find "./hadoop-hdds" -type f -executable | grep -v target | grep -v node_modules | grep -v py | xargs -n1 shellcheck | tee "$OUTPUT_FILE"
find "./hadoop-ozone" -type f -executable | grep -v target | grep -v node_modules | grep -v py | xargs -n1 shellcheck | tee "$OUTPUT_FILE"
if [ "$(cat "$OUTPUT_FILE")" ]; then
exit 1
fi

8
hadoop-ozone/dev-support/checks/unit.sh
View File

@ -14,11 +14,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
export MAVEN_OPTS="-Xmx4096m"
mvn -fn test -am -pl :hadoop-ozone-dist -P hdds
module_failed_tests=$(find "." -name 'TEST*.xml'\
| xargs "grep" -l -E "<failure|<error"\
mvn -fn test -f pom.ozone.xml -pl \!:hadoop-ozone-integration-test,\!:hadoop-ozone-filesystem
module_failed_tests=$(find "." -name 'TEST*.xml' -print0 \
| xargs -n1 -0 "grep" -l -E "<failure|<error"\
| awk -F/ '{sub("'"TEST-JUNIT_TEST_OUTPUT_DIR"'",""); sub(".xml",""); print $NF}')
if [[ -n "${module_failed_tests}" ]] ; then
exit -1
exit 1
fi
exit 0