diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile index 9e1c7699e5..b9e56fd205 100644 --- a/dev-support/Jenkinsfile +++ b/dev-support/Jenkinsfile @@ -59,6 +59,13 @@ pipeline { // optional stages after the first one. stage ('setup sources') { steps { + dir("${WORKSPACE}/centos-7") { + sh '''#!/usr/bin/env bash + + cp -Rp ${WORKSPACE}/src ${WORKSPACE}/centos-7 + ''' + } + dir("${WORKSPACE}/centos-8") { sh '''#!/usr/bin/env bash @@ -82,6 +89,58 @@ pipeline { } } + // This is an optional stage which runs only when there's a change in + // C++/C++ build/platform. + // This stage serves as a means of cross platform validation, which is + // really needed to ensure that any C++ related/platform change doesn't + // break the Hadoop build on Centos 7. + stage ('precommit-run Centos 7') { + environment { + SOURCEDIR = "${WORKSPACE}/centos-7/src" + PATCHDIR = "${WORKSPACE}/centos-7/out" + DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_7" + IS_OPTIONAL = 1 + } + + steps { + withCredentials( + [usernamePassword(credentialsId: 'apache-hadoop-at-github.com', + passwordVariable: 'GITHUB_TOKEN', + usernameVariable: 'GITHUB_USER'), + usernamePassword(credentialsId: 'hadoopqa-at-asf-jira', + passwordVariable: 'JIRA_PASSWORD', + usernameVariable: 'JIRA_USER')]) { + sh '''#!/usr/bin/env bash + + chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" + "${SOURCEDIR}/dev-support/jenkins.sh" run_ci + ''' + } + } + + post { + // Since this is an optional platform, we want to copy the artifacts + // and archive it only if the build fails, to help with debugging. + failure { + sh '''#!/usr/bin/env bash + + cp -Rp "${WORKSPACE}/centos-7/out" "${WORKSPACE}" + ''' + archiveArtifacts "out/**" + } + + cleanup() { + script { + sh '''#!/usr/bin/env bash + + chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh" + "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc + ''' + } + } + } + } + // This is an optional stage which runs only when there's a change in // C++/C++ build/platform. // This stage serves as a means of cross platform validation, which is diff --git a/dev-support/docker/Dockerfile_centos_7 b/dev-support/docker/Dockerfile_centos_7 index b645d81c29..ccb445be26 100644 --- a/dev-support/docker/Dockerfile_centos_7 +++ b/dev-support/docker/Dockerfile_centos_7 @@ -35,6 +35,7 @@ RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \ ###### # hadolint ignore=DL3008,SC2046 RUN yum update -y \ + && yum groupinstall -y "Development Tools" \ && yum install -y \ centos-release-scl \ python3 \ @@ -44,6 +45,21 @@ RUN yum update -y \ RUN echo "source /opt/rh/devtoolset-9/enable" >> /etc/bashrc SHELL ["/bin/bash", "--login", "-c"] +###### +# Set the environment variables needed for CMake +# to find and use GCC 9 for compilation +###### +ENV GCC_HOME "/opt/rh/devtoolset-9" +ENV CC "${GCC_HOME}/root/usr/bin/gcc" +ENV CXX "${GCC_HOME}/root/usr/bin/g++" +ENV SHLVL 1 +ENV LD_LIBRARY_PATH "${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:${GCC_HOME}/root/usr/lib64/dyninst:${GCC_HOME}/root/usr/lib/dyninst:${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:/usr/lib:/usr/lib64" +ENV PCP_DIR "${GCC_HOME}/root" +ENV MANPATH "${GCC_HOME}/root/usr/share/man:" +ENV PATH "${GCC_HOME}/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +ENV PKG_CONFIG_PATH "${GCC_HOME}/root/usr/lib64/pkgconfig" +ENV INFOPATH "${GCC_HOME}/root/usr/share/info" + # TODO: Set locale ###### @@ -65,7 +81,6 @@ ENV SPOTBUGS_HOME /opt/spotbugs ENV PROTOBUF_HOME /opt/protobuf ENV PATH "${PATH}:/opt/protobuf/bin" - ###### # Install packages ###### @@ -76,4 +91,6 @@ RUN pkg-resolver/install-yasm.sh centos:7 RUN pkg-resolver/install-protobuf.sh centos:7 RUN pkg-resolver/install-boost.sh centos:7 RUN pkg-resolver/install-spotbugs.sh centos:7 +RUN pkg-resolver/install-nodejs.sh centos:7 +RUN pkg-resolver/install-git.sh centos:7 RUN pkg-resolver/install-common-pkgs.sh diff --git a/dev-support/docker/pkg-resolver/install-git.sh b/dev-support/docker/pkg-resolver/install-git.sh new file mode 100644 index 0000000000..3536418198 --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-git.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +default_version="2.9.5" +version_to_install=$default_version +if [ -n "$2" ]; then + version_to_install="$2" +fi + +if [ "$version_to_install" != "2.9.5" ]; then + echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" + version_to_install=$default_version +fi + +if [ "$version_to_install" == "2.9.5" ]; then + # hadolint ignore=DL3003 + mkdir -p /tmp/git /opt/git && + curl -L -s -S https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.9.5.tar.gz >/tmp/git/git-2.9.5.tar.gz && + tar xzf /tmp/git/git-2.9.5.tar.gz --strip-components 1 -C /opt/git && + cd /opt/git || exit && + make configure && + ./configure --prefix=/usr/local && + make "-j$(nproc)" && + make install && + cd /root || exit +else + echo "ERROR: Don't know how to install version $version_to_install" + exit 1 +fi diff --git a/dev-support/docker/pkg-resolver/install-nodejs.sh b/dev-support/docker/pkg-resolver/install-nodejs.sh new file mode 100644 index 0000000000..5ba1c22808 --- /dev/null +++ b/dev-support/docker/pkg-resolver/install-nodejs.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; then + echo "ERROR: Need at least 1 argument, $# were provided" + exit 1 +fi + +pkg-resolver/check_platform.py "$1" +if [ $? -eq 1 ]; then + echo "ERROR: Unsupported platform $1" + exit 1 +fi + +default_version="14.16.1" +version_to_install=$default_version +if [ -n "$2" ]; then + version_to_install="$2" +fi + +if [ "$version_to_install" != "14.16.1" ]; then + echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" + version_to_install=$default_version +fi + +if [ "$version_to_install" == "14.16.1" ]; then + # hadolint ignore=DL3003 + mkdir -p /tmp/node && + curl -L -s -S https://nodejs.org/dist/v14.16.1/node-v14.16.1.tar.gz -o /tmp/node-v14.16.1.tar.gz && + tar xzf /tmp/node-v14.16.1.tar.gz --strip-components 1 -C /tmp/node && + cd /tmp/node || exit && + ./configure && + make "-j$(nproc)" && + make install && + cd /root || exit +else + echo "ERROR: Don't know how to install version $version_to_install" + exit 1 +fi diff --git a/dev-support/docker/pkg-resolver/packages.json b/dev-support/docker/pkg-resolver/packages.json index e794b4cfdb..f84196fa70 100644 --- a/dev-support/docker/pkg-resolver/packages.json +++ b/dev-support/docker/pkg-resolver/packages.json @@ -18,6 +18,9 @@ "centos:7": "automake", "centos:8": "automake" }, + "autoconf": { + "centos:7": "autoconf" + }, "bats": { "debian:10": "bats", "ubuntu:focal": "bats", @@ -136,11 +139,13 @@ "devtoolset-9" ] }, + "gettext": { + "centos:7": "gettext-devel" + }, "git": { "debian:10": "git", "ubuntu:focal": "git", "ubuntu:focal::arch64": "git", - "centos:7": "git", "centos:8": "git" }, "gnupg-agent": { @@ -172,6 +177,12 @@ "centos:7": "openssl-devel", "centos:8": "openssl-devel" }, + "perl": { + "centos:7": [ + "perl-CPAN", + "perl-devel" + ] + }, "protocol-buffers": { "debian:10": [ "libprotobuf-dev",