HADOOP-17807. Use separate src dir for platform builds (#3210)
This commit is contained in:
Gautham B A
GitHub
parent
d710ec8d85
commit
97c88c97de
139
dev-support/Jenkinsfile
vendored
139
dev-support/Jenkinsfile
vendored
@ -29,9 +29,6 @@ pipeline {
|
||||
}
|
||||
|
||||
environment {
|
||||
SOURCEDIR = 'src'
|
||||
// will also need to change notification section below
|
||||
PATCHDIR = 'out'
|
||||
YETUS='yetus'
|
||||
// Branch or tag name. Yetus release tags are 'rel/X.Y.Z'
|
||||
YETUS_VERSION='f9ba0170a5787a5f4662d3769804fef0226a182f'
|
||||
@ -56,6 +53,35 @@ pipeline {
|
||||
}
|
||||
}
|
||||
|
||||
// Setup codebase so that each platform's build happens in its own exclusive copy of the
|
||||
// codebase.
|
||||
// Primarily because YETUS messes up the git branch information and affects the subsequent
|
||||
// optional stages after the first one.
|
||||
stage ('setup sources') {
|
||||
steps {
|
||||
dir("${WORKSPACE}/centos-8") {
|
||||
sh '''#!/usr/bin/env bash
|
||||
|
||||
cp -Rp ${WORKSPACE}/src ${WORKSPACE}/centos-8
|
||||
'''
|
||||
}
|
||||
|
||||
dir("${WORKSPACE}/debian-10") {
|
||||
sh '''#!/usr/bin/env bash
|
||||
|
||||
cp -Rp ${WORKSPACE}/src ${WORKSPACE}/debian-10
|
||||
'''
|
||||
}
|
||||
|
||||
dir("${WORKSPACE}/ubuntu-focal") {
|
||||
sh '''#!/usr/bin/env bash
|
||||
|
||||
cp -Rp ${WORKSPACE}/src ${WORKSPACE}/ubuntu-focal
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// This is an optional stage which runs only when there's a change in
|
||||
// C++/C++ build/platform.
|
||||
// This stage serves as a means of cross platform validation, which is
|
||||
@ -63,6 +89,8 @@ pipeline {
|
||||
// break the Hadoop build on Centos 8.
|
||||
stage ('precommit-run Centos 8') {
|
||||
environment {
|
||||
SOURCEDIR = "${WORKSPACE}/centos-8/src"
|
||||
PATCHDIR = "${WORKSPACE}/centos-8/out"
|
||||
DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_8"
|
||||
IS_OPTIONAL = 1
|
||||
}
|
||||
@ -78,10 +106,32 @@ pipeline {
|
||||
sh '''#!/usr/bin/env bash
|
||||
|
||||
chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
|
||||
"${SOURCEDIR}/dev-support/jenkins.sh"
|
||||
"${SOURCEDIR}/dev-support/jenkins.sh" run_ci
|
||||
'''
|
||||
}
|
||||
}
|
||||
|
||||
post {
|
||||
// Since this is an optional platform, we want to copy the artifacts
|
||||
// and archive it only if the build fails, to help with debugging.
|
||||
failure {
|
||||
sh '''#!/usr/bin/env bash
|
||||
|
||||
cp -Rp "${WORKSPACE}/centos-8/out" "${WORKSPACE}"
|
||||
'''
|
||||
archiveArtifacts "out/**"
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
script {
|
||||
sh '''#!/usr/bin/env bash
|
||||
|
||||
chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
|
||||
"${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// This is an optional stage which runs only when there's a change in
|
||||
@ -91,6 +141,8 @@ pipeline {
|
||||
// break the Hadoop build on Debian 10.
|
||||
stage ('precommit-run Debian 10') {
|
||||
environment {
|
||||
SOURCEDIR = "${WORKSPACE}/debian-10/src"
|
||||
PATCHDIR = "${WORKSPACE}/debian-10/out"
|
||||
DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_10"
|
||||
IS_OPTIONAL = 1
|
||||
}
|
||||
@ -106,16 +158,40 @@ pipeline {
|
||||
sh '''#!/usr/bin/env bash
|
||||
|
||||
chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
|
||||
"${SOURCEDIR}/dev-support/jenkins.sh"
|
||||
"${SOURCEDIR}/dev-support/jenkins.sh" run_ci
|
||||
'''
|
||||
}
|
||||
}
|
||||
|
||||
post {
|
||||
// Since this is an optional platform, we want to copy the artifacts
|
||||
// and archive it only if the build fails, to help with debugging.
|
||||
failure {
|
||||
sh '''#!/usr/bin/env bash
|
||||
|
||||
cp -Rp "${WORKSPACE}/debian-10/out" "${WORKSPACE}"
|
||||
'''
|
||||
archiveArtifacts "out/**"
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
script {
|
||||
sh '''#!/usr/bin/env bash
|
||||
|
||||
chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
|
||||
"${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// We want to use Ubuntu Focal as our main CI and thus, this stage
|
||||
// isn't optional (runs for all the PRs).
|
||||
stage ('precommit-run Ubuntu focal') {
|
||||
environment {
|
||||
SOURCEDIR = "${WORKSPACE}/ubuntu-focal/src"
|
||||
PATCHDIR = "${WORKSPACE}/ubuntu-focal/out"
|
||||
DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile"
|
||||
IS_OPTIONAL = 0
|
||||
}
|
||||
@ -131,13 +207,10 @@ pipeline {
|
||||
sh '''#!/usr/bin/env bash
|
||||
|
||||
chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
|
||||
"${SOURCEDIR}/dev-support/jenkins.sh"
|
||||
"${SOURCEDIR}/dev-support/jenkins.sh" run_ci
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
post {
|
||||
always {
|
||||
@ -148,15 +221,19 @@ pipeline {
|
||||
passwordVariable: 'GITHUB_TOKEN',
|
||||
usernameVariable: 'GITHUB_USER')]) {
|
||||
sh '''#!/usr/bin/env bash
|
||||
YETUS_ARGS+=("--github-token=${GITHUB_TOKEN}")
|
||||
YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}")
|
||||
TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/github-status-recovery.sh"
|
||||
/usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" ${EXTRA_ARGS} || true
|
||||
|
||||
# Copy the artifacts of Ubuntu focal build to workspace
|
||||
cp -Rp "${WORKSPACE}/ubuntu-focal/out" "${WORKSPACE}"
|
||||
|
||||
# Send Github status
|
||||
chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
|
||||
"${SOURCEDIR}/dev-support/jenkins.sh" github_status_recovery
|
||||
'''
|
||||
}
|
||||
|
||||
// Yetus output
|
||||
archiveArtifacts "${env.PATCHDIR}/**"
|
||||
// YETUS output
|
||||
archiveArtifacts "out/**"
|
||||
|
||||
// Publish the HTML report so that it can be looked at
|
||||
// Has to be relative to WORKSPACE.
|
||||
publishHTML (target: [
|
||||
@ -164,33 +241,39 @@ pipeline {
|
||||
keepAll: true,
|
||||
alwaysLinkToLastBuild: true,
|
||||
// Has to be relative to WORKSPACE
|
||||
reportDir: "${env.PATCHDIR}",
|
||||
reportDir: "out",
|
||||
reportFiles: 'report.html',
|
||||
reportName: 'Yetus Report'
|
||||
])
|
||||
|
||||
// Publish JUnit results
|
||||
try {
|
||||
junit "${env.SOURCEDIR}/**/target/surefire-reports/*.xml"
|
||||
junit "${SOURCEDIR}/**/target/surefire-reports/*.xml"
|
||||
} catch(e) {
|
||||
echo 'junit processing: ' + e.toString()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
script {
|
||||
sh '''#!/usr/bin/env bash
|
||||
|
||||
chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
|
||||
"${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
post {
|
||||
// Jenkins pipeline jobs fill slaves on PRs without this :(
|
||||
cleanup() {
|
||||
script {
|
||||
sh '''
|
||||
# See YETUS-764
|
||||
if [ -f "${WORKSPACE}/${PATCHDIR}/pidfile.txt" ]; then
|
||||
echo "test-patch process appears to still be running: killing"
|
||||
kill `cat "${WORKSPACE}/${PATCHDIR}/pidfile.txt"` || true
|
||||
sleep 10
|
||||
fi
|
||||
if [ -f "${WORKSPACE}/${PATCHDIR}/cidfile.txt" ]; then
|
||||
echo "test-patch container appears to still be running: killing"
|
||||
docker kill `cat "${WORKSPACE}/${PATCHDIR}/cidfile.txt"` || true
|
||||
fi
|
||||
sh '''#!/usr/bin/env bash
|
||||
|
||||
# See HADOOP-13951
|
||||
chmod -R u+rxw "${WORKSPACE}"
|
||||
'''
|
||||
|
||||
@ -115,10 +115,10 @@ function run_ci() {
|
||||
TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/test-patch.sh"
|
||||
|
||||
# this must be clean for every run
|
||||
if [[ -d "${WORKSPACE}/${PATCHDIR}" ]]; then
|
||||
rm -rf "${WORKSPACE:?}/${PATCHDIR}"
|
||||
if [[ -d "${PATCHDIR}" ]]; then
|
||||
rm -rf "${PATCHDIR:?}"
|
||||
fi
|
||||
mkdir -p "${WORKSPACE}/${PATCHDIR}"
|
||||
mkdir -p "${PATCHDIR}"
|
||||
|
||||
# if given a JIRA issue, process it. If CHANGE_URL is set
|
||||
# (e.g., Github Branch Source plugin), process it.
|
||||
@ -128,23 +128,23 @@ function run_ci() {
|
||||
if [[ -n "${JIRA_ISSUE_KEY}" ]]; then
|
||||
YETUS_ARGS+=("${JIRA_ISSUE_KEY}")
|
||||
elif [[ -z "${CHANGE_URL}" ]]; then
|
||||
echo "Full build skipped" >"${WORKSPACE}/${PATCHDIR}/report.html"
|
||||
echo "Full build skipped" >"${PATCHDIR}/report.html"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}")
|
||||
YETUS_ARGS+=("--patch-dir=${PATCHDIR}")
|
||||
|
||||
# where the source is located
|
||||
YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}")
|
||||
YETUS_ARGS+=("--basedir=${SOURCEDIR}")
|
||||
|
||||
# our project defaults come from a personality file
|
||||
YETUS_ARGS+=("--project=hadoop")
|
||||
YETUS_ARGS+=("--personality=${WORKSPACE}/${SOURCEDIR}/dev-support/bin/hadoop.sh")
|
||||
YETUS_ARGS+=("--personality=${SOURCEDIR}/dev-support/bin/hadoop.sh")
|
||||
|
||||
# lots of different output formats
|
||||
YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt")
|
||||
YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt")
|
||||
YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html")
|
||||
YETUS_ARGS+=("--brief-report-file=${PATCHDIR}/brief.txt")
|
||||
YETUS_ARGS+=("--console-report-file=${PATCHDIR}/console.txt")
|
||||
YETUS_ARGS+=("--html-report-file=${PATCHDIR}/report.html")
|
||||
|
||||
# enable writing back to Github
|
||||
YETUS_ARGS+=("--github-token=${GITHUB_TOKEN}")
|
||||
@ -206,7 +206,49 @@ function run_ci() {
|
||||
"${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
|
||||
}
|
||||
|
||||
# Check if the CI needs to be run, if so, do so :)
|
||||
if check_ci_run; then
|
||||
run_ci
|
||||
## @description Cleans up the processes started by YETUS
|
||||
function cleanup_ci_proc() {
|
||||
# See YETUS-764
|
||||
if [ -f "${PATCHDIR}/pidfile.txt" ]; then
|
||||
echo "test-patch process appears to still be running: killing"
|
||||
kill "$(cat "${PATCHDIR}/pidfile.txt")" || true
|
||||
sleep 10
|
||||
fi
|
||||
if [ -f "${PATCHDIR}/cidfile.txt" ]; then
|
||||
echo "test-patch container appears to still be running: killing"
|
||||
docker kill "$(cat "${PATCHDIR}/cidfile.txt")" || true
|
||||
fi
|
||||
}
|
||||
|
||||
## @description Invokes github_status_recovery in YETUS's precommit
|
||||
function github_status_recovery() {
|
||||
YETUS_ARGS+=("--github-token=${GITHUB_TOKEN}")
|
||||
YETUS_ARGS+=("--patch-dir=${PATCHDIR}")
|
||||
TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/github-status-recovery.sh"
|
||||
/usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" "${EXTRA_ARGS}" || true
|
||||
}
|
||||
|
||||
if [ -z "$1" ]; then
|
||||
echo "Must specify an argument for jenkins.sh"
|
||||
echo "run_ci - Runs the CI based on platform image as defined by DOCKERFILE"
|
||||
echo "cleanup_ci_proc - Cleans up the processes spawned for running the CI"
|
||||
echo "github_status_recovery - Sends Github status (refer to YETUS precommit for more details)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Process arguments to jenkins.sh
|
||||
if [ "$1" == "run_ci" ]; then
|
||||
# Check if the CI needs to be run, if so, do so :)
|
||||
if check_ci_run; then
|
||||
run_ci
|
||||
else
|
||||
echo "No C++ file/C++ build/platform changes found, will not run CI"
|
||||
fi
|
||||
elif [ "$1" == "cleanup_ci_proc" ]; then
|
||||
cleanup_ci_proc
|
||||
elif [ "$1" == "github_status_recovery" ]; then
|
||||
github_status_recovery
|
||||
else
|
||||
echo "Don't know how to process $1"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
Loading…
Reference in New Issue
Block a user