diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index 6cabbfe7b5..9e1c7699e5 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -29,9 +29,6 @@ pipeline { } environment { - SOURCEDIR = 'src' - // will also need to change notification section below - PATCHDIR = 'out' YETUS='yetus' // Branch or tag name. Yetus release tags are 'rel/X.Y.Z' YETUS_VERSION='f9ba0170a5787a5f4662d3769804fef0226a182f' @@ -56,6 +53,35 @@ pipeline { } } + // Setup codebase so that each platform's build happens in its own exclusive copy of the + // codebase. + // Primarily because YETUS messes up the git branch information and affects the subsequent + // optional stages after the first one. + stage ('setup sources') { + steps { + dir("${WORKSPACE}/centos-8") { + sh '''#!/usr/bin/env bash + + cp -Rp ${WORKSPACE}/src ${WORKSPACE}/centos-8 + ''' + } + + dir("${WORKSPACE}/debian-10") { + sh '''#!/usr/bin/env bash + + cp -Rp ${WORKSPACE}/src ${WORKSPACE}/debian-10 + ''' + } + + dir("${WORKSPACE}/ubuntu-focal") { + sh '''#!/usr/bin/env bash + + cp -Rp ${WORKSPACE}/src ${WORKSPACE}/ubuntu-focal + ''' + } + } + } + // This is an optional stage which runs only when there's a change in // C++/C++ build/platform. // This stage serves as a means of cross platform validation, which is @@ -63,6 +89,8 @@ pipeline { // break the Hadoop build on Centos 8. stage ('precommit-run Centos 8') { environment { + SOURCEDIR = "${WORKSPACE}/centos-8/src" + PATCHDIR = "${WORKSPACE}/centos-8/out" DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_8" IS_OPTIONAL = 1 } @@ -78,10 +106,32 @@ pipeline { sh '''#!/usr/bin/env bash chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" - "${SOURCEDIR}/dev-support/jenkins.sh" + "${SOURCEDIR}/dev-support/jenkins.sh" run_ci ''' } } + + post { + // Since this is an optional platform, we want to copy the artifacts + // and archive it only if the build fails, to help with debugging. + failure { + sh '''#!/usr/bin/env bash + + cp -Rp "${WORKSPACE}/centos-8/out" "${WORKSPACE}" + ''' + archiveArtifacts "out/**" + } + + cleanup() { + script { + sh '''#!/usr/bin/env bash + + chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" + "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc + ''' + } + } + } } // This is an optional stage which runs only when there's a change in @@ -91,6 +141,8 @@ pipeline { // break the Hadoop build on Debian 10. stage ('precommit-run Debian 10') { environment { + SOURCEDIR = "${WORKSPACE}/debian-10/src" + PATCHDIR = "${WORKSPACE}/debian-10/out" DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_10" IS_OPTIONAL = 1 } @@ -106,16 +158,40 @@ pipeline { sh '''#!/usr/bin/env bash chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" - "${SOURCEDIR}/dev-support/jenkins.sh" + "${SOURCEDIR}/dev-support/jenkins.sh" run_ci ''' } } + + post { + // Since this is an optional platform, we want to copy the artifacts + // and archive it only if the build fails, to help with debugging. + failure { + sh '''#!/usr/bin/env bash + + cp -Rp "${WORKSPACE}/debian-10/out" "${WORKSPACE}" + ''' + archiveArtifacts "out/**" + } + + cleanup() { + script { + sh '''#!/usr/bin/env bash + + chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" + "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc + ''' + } + } + } } // We want to use Ubuntu Focal as our main CI and thus, this stage // isn't optional (runs for all the PRs). stage ('precommit-run Ubuntu focal') { environment { + SOURCEDIR = "${WORKSPACE}/ubuntu-focal/src" + PATCHDIR = "${WORKSPACE}/ubuntu-focal/out" DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile" IS_OPTIONAL = 0 } @@ -131,69 +207,76 @@ pipeline { sh '''#!/usr/bin/env bash chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" - "${SOURCEDIR}/dev-support/jenkins.sh" + "${SOURCEDIR}/dev-support/jenkins.sh" run_ci ''' } } - } + post { + always { + script { + // Publish status if it was missed (YETUS-1059) + withCredentials( + [usernamePassword(credentialsId: '683f5dcf-5552-4b28-9fb1-6a6b77cf53dd', + passwordVariable: 'GITHUB_TOKEN', + usernameVariable: 'GITHUB_USER')]) { + sh '''#!/usr/bin/env bash + + # Copy the artifacts of Ubuntu focal build to workspace + cp -Rp "${WORKSPACE}/ubuntu-focal/out" "${WORKSPACE}" + + # Send Github status + chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" + "${SOURCEDIR}/dev-support/jenkins.sh" github_status_recovery + ''' + } + + // YETUS output + archiveArtifacts "out/**" + + // Publish the HTML report so that it can be looked at + // Has to be relative to WORKSPACE. + publishHTML (target: [ + allowMissing: true, + keepAll: true, + alwaysLinkToLastBuild: true, + // Has to be relative to WORKSPACE + reportDir: "out", + reportFiles: 'report.html', + reportName: 'Yetus Report' + ]) + + // Publish JUnit results + try { + junit "${SOURCEDIR}/**/target/surefire-reports/*.xml" + } catch(e) { + echo 'junit processing: ' + e.toString() + } + } + } + + cleanup() { + script { + sh '''#!/usr/bin/env bash + + chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" + "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc + ''' + } + } + } + } } post { - always { - script { - // Publish status if it was missed (YETUS-1059) - withCredentials( - [usernamePassword(credentialsId: '683f5dcf-5552-4b28-9fb1-6a6b77cf53dd', - passwordVariable: 'GITHUB_TOKEN', - usernameVariable: 'GITHUB_USER')]) { - sh '''#!/usr/bin/env bash - YETUS_ARGS+=("--github-token=${GITHUB_TOKEN}") - YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}") - TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/github-status-recovery.sh" - /usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" ${EXTRA_ARGS} || true - ''' - } - - // Yetus output - archiveArtifacts "${env.PATCHDIR}/**" - // Publish the HTML report so that it can be looked at - // Has to be relative to WORKSPACE. - publishHTML (target: [ - allowMissing: true, - keepAll: true, - alwaysLinkToLastBuild: true, - // Has to be relative to WORKSPACE - reportDir: "${env.PATCHDIR}", - reportFiles: 'report.html', - reportName: 'Yetus Report' - ]) - // Publish JUnit results - try { - junit "${env.SOURCEDIR}/**/target/surefire-reports/*.xml" - } catch(e) { - echo 'junit processing: ' + e.toString() - } - } - } - // Jenkins pipeline jobs fill slaves on PRs without this :( cleanup() { script { - sh ''' - # See YETUS-764 - if [ -f "${WORKSPACE}/${PATCHDIR}/pidfile.txt" ]; then - echo "test-patch process appears to still be running: killing" - kill `cat "${WORKSPACE}/${PATCHDIR}/pidfile.txt"` || true - sleep 10 - fi - if [ -f "${WORKSPACE}/${PATCHDIR}/cidfile.txt" ]; then - echo "test-patch container appears to still be running: killing" - docker kill `cat "${WORKSPACE}/${PATCHDIR}/cidfile.txt"` || true - fi - # See HADOOP-13951 - chmod -R u+rxw "${WORKSPACE}" - ''' + sh '''#!/usr/bin/env bash + + # See HADOOP-13951 + chmod -R u+rxw "${WORKSPACE}" + ''' deleteDir() } } diff --git a/dev-support/jenkins.sh b/dev-support/jenkins.sh index 09905dd320..35431fbe5b 100644 --- a/dev-support/jenkins.sh +++ b/dev-support/jenkins.sh @@ -115,10 +115,10 @@ function run_ci() { TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/test-patch.sh" # this must be clean for every run - if [[ -d "${WORKSPACE}/${PATCHDIR}" ]]; then - rm -rf "${WORKSPACE:?}/${PATCHDIR}" + if [[ -d "${PATCHDIR}" ]]; then + rm -rf "${PATCHDIR:?}" fi - mkdir -p "${WORKSPACE}/${PATCHDIR}" + mkdir -p "${PATCHDIR}" # if given a JIRA issue, process it. If CHANGE_URL is set # (e.g., Github Branch Source plugin), process it. @@ -128,23 +128,23 @@ function run_ci() { if [[ -n "${JIRA_ISSUE_KEY}" ]]; then YETUS_ARGS+=("${JIRA_ISSUE_KEY}") elif [[ -z "${CHANGE_URL}" ]]; then - echo "Full build skipped" >"${WORKSPACE}/${PATCHDIR}/report.html" + echo "Full build skipped" >"${PATCHDIR}/report.html" exit 0 fi - YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}") + YETUS_ARGS+=("--patch-dir=${PATCHDIR}") # where the source is located - YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}") + YETUS_ARGS+=("--basedir=${SOURCEDIR}") # our project defaults come from a personality file YETUS_ARGS+=("--project=hadoop") - YETUS_ARGS+=("--personality=${WORKSPACE}/${SOURCEDIR}/dev-support/bin/hadoop.sh") + YETUS_ARGS+=("--personality=${SOURCEDIR}/dev-support/bin/hadoop.sh") # lots of different output formats - YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt") - YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt") - YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html") + YETUS_ARGS+=("--brief-report-file=${PATCHDIR}/brief.txt") + YETUS_ARGS+=("--console-report-file=${PATCHDIR}/console.txt") + YETUS_ARGS+=("--html-report-file=${PATCHDIR}/report.html") # enable writing back to Github YETUS_ARGS+=("--github-token=${GITHUB_TOKEN}") @@ -206,7 +206,49 @@ function run_ci() { "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" } -# Check if the CI needs to be run, if so, do so :) -if check_ci_run; then - run_ci +## @description Cleans up the processes started by YETUS +function cleanup_ci_proc() { + # See YETUS-764 + if [ -f "${PATCHDIR}/pidfile.txt" ]; then + echo "test-patch process appears to still be running: killing" + kill "$(cat "${PATCHDIR}/pidfile.txt")" || true + sleep 10 + fi + if [ -f "${PATCHDIR}/cidfile.txt" ]; then + echo "test-patch container appears to still be running: killing" + docker kill "$(cat "${PATCHDIR}/cidfile.txt")" || true + fi +} + +## @description Invokes github_status_recovery in YETUS's precommit +function github_status_recovery() { + YETUS_ARGS+=("--github-token=${GITHUB_TOKEN}") + YETUS_ARGS+=("--patch-dir=${PATCHDIR}") + TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/github-status-recovery.sh" + /usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" "${EXTRA_ARGS}" || true +} + +if [ -z "$1" ]; then + echo "Must specify an argument for jenkins.sh" + echo "run_ci - Runs the CI based on platform image as defined by DOCKERFILE" + echo "cleanup_ci_proc - Cleans up the processes spawned for running the CI" + echo "github_status_recovery - Sends Github status (refer to YETUS precommit for more details)" + exit 1 +fi + +# Process arguments to jenkins.sh +if [ "$1" == "run_ci" ]; then + # Check if the CI needs to be run, if so, do so :) + if check_ci_run; then + run_ci + else + echo "No C++ file/C++ build/platform changes found, will not run CI" + fi +elif [ "$1" == "cleanup_ci_proc" ]; then + cleanup_ci_proc +elif [ "$1" == "github_status_recovery" ]; then + github_status_recovery +else + echo "Don't know how to process $1" + exit 1 fi