#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # This script is called from the Jenkinsfile, which ultimately runs # the CI through Yetus. # We use Ubuntu Focal as the main platform for building Hadoop, thus # it runs for all the PRs. Additionally, we also ensure that # Hadoop builds across the supported platforms whenever there's a change # in any of the C++ files, C++ build files or platform changes. ## @description Check if the given extension is related to C++ ## @param seeking ## @return 0 if yes ## @return 1 if no is_cpp_extension() { local cpp_extensions=("cc" "cpp" "h" "hpp") local seeking=$1 for element in "${cpp_extensions[@]}"; do if [[ $element == "$seeking" ]]; then return 0 fi done return 1 } ## @description Check if the given relative path corresponds to ## change in platform files ## @param in_path ## @return 0 if yes ## @return 1 if no is_platform_change() { declare in_path in_path="${SOURCEDIR}"/"${1}" for path in "${SOURCEDIR}"/dev-support/docker/Dockerfile* "${SOURCEDIR}"/dev-support/docker/pkg-resolver/*.json; do if [ "${in_path}" == "${path}" ]; then echo "Found C++ platform related changes in ${in_path}" return 0 fi done return 1 } ## @description Checks if the given path corresponds to a change ## in C++ files or related to C++ build system ## @param path ## @return 0 if yes ## @return 1 if no is_cpp_change() { shopt -s nocasematch local path=$1 declare filename filename=$(basename -- "${path}") extension=${filename##*.} if is_cpp_extension "${extension}"; then echo "Found C++ changes in ${path}" return 0 fi if [[ $filename =~ CMakeLists\.txt ]]; then echo "Found C++ build related changes in ${path}" return 0 fi return 1 } ## @description Check if the CI needs to be run - CI will always run if ## IS_OPTIONAL is 0, or if there's any change in ## C++/C++ build/platform ## @return 0 if yes ## @return 1 if no function check_ci_run() { # Get the first commit of this PR relative to the trunk branch firstCommitOfThisPr=$(git --git-dir "${SOURCEDIR}/.git" rev-parse origin/trunk) # Loop over the paths of all the changed files and check if the criteria # to run the CI has been satisfied for path in $(git --git-dir "${SOURCEDIR}/.git" diff --name-only "${firstCommitOfThisPr}" HEAD); do if is_cpp_change "${path}"; then return 0 fi if is_platform_change "${path}"; then return 0 fi done # We must run the CI if it's not optional if [ "$IS_OPTIONAL" -eq 0 ]; then return 0 fi return 1 } ## @description Run the CI using YETUS function run_ci() { TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/test-patch.sh" # this must be clean for every run if [[ -d "${WORKSPACE}/${PATCHDIR}" ]]; then rm -rf "${WORKSPACE:?}/${PATCHDIR}" fi mkdir -p "${WORKSPACE}/${PATCHDIR}" # if given a JIRA issue, process it. If CHANGE_URL is set # (e.g., Github Branch Source plugin), process it. # otherwise exit, because we don't want Hadoop to do a # full build. We wouldn't normally do this check for smaller # projects. :) if [[ -n "${JIRA_ISSUE_KEY}" ]]; then YETUS_ARGS+=("${JIRA_ISSUE_KEY}") elif [[ -z "${CHANGE_URL}" ]]; then echo "Full build skipped" >"${WORKSPACE}/${PATCHDIR}/report.html" exit 0 fi YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}") # where the source is located YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}") # our project defaults come from a personality file YETUS_ARGS+=("--project=hadoop") YETUS_ARGS+=("--personality=${WORKSPACE}/${SOURCEDIR}/dev-support/bin/hadoop.sh") # lots of different output formats YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt") YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt") YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html") # enable writing back to Github YETUS_ARGS+=("--github-token=${GITHUB_TOKEN}") # enable writing back to ASF JIRA YETUS_ARGS+=("--jira-password=${JIRA_PASSWORD}") YETUS_ARGS+=("--jira-user=${JIRA_USER}") # auto-kill any surefire stragglers during unit test runs YETUS_ARGS+=("--reapermode=kill") # set relatively high limits for ASF machines # changing these to higher values may cause problems # with other jobs on systemd-enabled machines YETUS_ARGS+=("--proclimit=5500") YETUS_ARGS+=("--dockermemlimit=22g") # -1 spotbugs issues that show up prior to the patch being applied YETUS_ARGS+=("--spotbugs-strict-precheck") # rsync these files back into the archive dir YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,spotbugsXml.xml") # URL for user-side presentation in reports and such to our artifacts # (needs to match the archive bits below) YETUS_ARGS+=("--build-url-artifacts=artifact/out") # plugins to enable YETUS_ARGS+=("--plugins=all") # don't let these tests cause -1s because we aren't really paying that # much attention to them YETUS_ARGS+=("--tests-filter=checkstyle") # run in docker mode and specifically point to our # Dockerfile since we don't want to use the auto-pulled version. YETUS_ARGS+=("--docker") YETUS_ARGS+=("--dockerfile=${DOCKERFILE}") YETUS_ARGS+=("--mvn-custom-repos") # effectively treat dev-suport as a custom maven module YETUS_ARGS+=("--skip-dirs=dev-support") # help keep the ASF boxes clean YETUS_ARGS+=("--sentinel") # test with Java 8 and 11 YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-8-openjdk-amd64") YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-11-openjdk-amd64") YETUS_ARGS+=("--multijdktests=compile") # custom javadoc goals YETUS_ARGS+=("--mvn-javadoc-goals=process-sources,javadoc:javadoc-no-fork") # write Yetus report as GitHub comment (YETUS-1102) YETUS_ARGS+=("--github-write-comment") YETUS_ARGS+=("--github-use-emoji-vote") "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" } # Check if the CI needs to be run, if so, do so :) if check_ci_run; then run_ci fi