hadoop/dev-support/Jenkinsfile
2021-06-30 22:04:54 -07:00

202 lines
7.9 KiB
Groovy

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
pipeline {
agent {
label 'Hadoop'
}
options {
buildDiscarder(logRotator(numToKeepStr: '5'))
timeout (time: 24, unit: 'HOURS')
timestamps()
checkoutToSubdirectory('src')
}
environment {
SOURCEDIR = 'src'
// will also need to change notification section below
PATCHDIR = 'out'
YETUS='yetus'
// Branch or tag name. Yetus release tags are 'rel/X.Y.Z'
YETUS_VERSION='f9ba0170a5787a5f4662d3769804fef0226a182f'
}
parameters {
string(name: 'JIRA_ISSUE_KEY',
defaultValue: '',
description: 'The JIRA issue that has a patch needing pre-commit testing. Example: HADOOP-1234')
}
stages {
stage ('install yetus') {
steps {
dir("${WORKSPACE}/${YETUS}") {
checkout([
$class: 'GitSCM',
branches: [[name: "${env.YETUS_VERSION}"]],
userRemoteConfigs: [[ url: 'https://github.com/apache/yetus.git']]]
)
}
}
}
// This is an optional stage which runs only when there's a change in
// C++/C++ build/platform.
// This stage serves as a means of cross platform validation, which is
// really needed to ensure that any C++ related/platform change doesn't
// break the Hadoop build on Centos 8.
stage ('precommit-run Centos 8') {
environment {
DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_8"
IS_OPTIONAL = 1
}
steps {
withCredentials(
[usernamePassword(credentialsId: 'apache-hadoop-at-github.com',
passwordVariable: 'GITHUB_TOKEN',
usernameVariable: 'GITHUB_USER'),
usernamePassword(credentialsId: 'hadoopqa-at-asf-jira',
passwordVariable: 'JIRA_PASSWORD',
usernameVariable: 'JIRA_USER')]) {
sh '''#!/usr/bin/env bash
chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
"${SOURCEDIR}/dev-support/jenkins.sh"
'''
}
}
}
// This is an optional stage which runs only when there's a change in
// C++/C++ build/platform.
// This stage serves as a means of cross platform validation, which is
// really needed to ensure that any C++ related/platform change doesn't
// break the Hadoop build on Debian 10.
stage ('precommit-run Debian 10') {
environment {
DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_10"
IS_OPTIONAL = 1
}
steps {
withCredentials(
[usernamePassword(credentialsId: 'apache-hadoop-at-github.com',
passwordVariable: 'GITHUB_TOKEN',
usernameVariable: 'GITHUB_USER'),
usernamePassword(credentialsId: 'hadoopqa-at-asf-jira',
passwordVariable: 'JIRA_PASSWORD',
usernameVariable: 'JIRA_USER')]) {
sh '''#!/usr/bin/env bash
chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
"${SOURCEDIR}/dev-support/jenkins.sh"
'''
}
}
}
// We want to use Ubuntu Focal as our main CI and thus, this stage
// isn't optional (runs for all the PRs).
stage ('precommit-run Ubuntu focal') {
environment {
DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile"
IS_OPTIONAL = 0
}
steps {
withCredentials(
[usernamePassword(credentialsId: 'apache-hadoop-at-github.com',
passwordVariable: 'GITHUB_TOKEN',
usernameVariable: 'GITHUB_USER'),
usernamePassword(credentialsId: 'hadoopqa-at-asf-jira',
passwordVariable: 'JIRA_PASSWORD',
usernameVariable: 'JIRA_USER')]) {
sh '''#!/usr/bin/env bash
chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
"${SOURCEDIR}/dev-support/jenkins.sh"
'''
}
}
}
}
post {
always {
script {
// Publish status if it was missed (YETUS-1059)
withCredentials(
[usernamePassword(credentialsId: '683f5dcf-5552-4b28-9fb1-6a6b77cf53dd',
passwordVariable: 'GITHUB_TOKEN',
usernameVariable: 'GITHUB_USER')]) {
sh '''#!/usr/bin/env bash
YETUS_ARGS+=("--github-token=${GITHUB_TOKEN}")
YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}")
TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/github-status-recovery.sh"
/usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" ${EXTRA_ARGS} || true
'''
}
// Yetus output
archiveArtifacts "${env.PATCHDIR}/**"
// Publish the HTML report so that it can be looked at
// Has to be relative to WORKSPACE.
publishHTML (target: [
allowMissing: true,
keepAll: true,
alwaysLinkToLastBuild: true,
// Has to be relative to WORKSPACE
reportDir: "${env.PATCHDIR}",
reportFiles: 'report.html',
reportName: 'Yetus Report'
])
// Publish JUnit results
try {
junit "${env.SOURCEDIR}/**/target/surefire-reports/*.xml"
} catch(e) {
echo 'junit processing: ' + e.toString()
}
}
}
// Jenkins pipeline jobs fill slaves on PRs without this :(
cleanup() {
script {
sh '''
# See YETUS-764
if [ -f "${WORKSPACE}/${PATCHDIR}/pidfile.txt" ]; then
echo "test-patch process appears to still be running: killing"
kill `cat "${WORKSPACE}/${PATCHDIR}/pidfile.txt"` || true
sleep 10
fi
if [ -f "${WORKSPACE}/${PATCHDIR}/cidfile.txt" ]; then
echo "test-patch container appears to still be running: killing"
docker kill `cat "${WORKSPACE}/${PATCHDIR}/cidfile.txt"` || true
fi
# See HADOOP-13951
chmod -R u+rxw "${WORKSPACE}"
'''
deleteDir()
}
}
}
}