HADOOP-18134. Setup Jenkins nightly CI for Windows 10 (#5062)

This PR gets Yetus to run on Windows 10
against the Hadoop codebase. It introduces
the following changes to allow us to setup
the nightly CI on Jenkins for Hadoop on
Windows 10.
* Hadoop personality changes for Yetus.
  Additional arguments have been passed,
  which are necessary to build and test
  Hadoop on Windows 10.
* Docker image for building Hadoop on
  Windows 10.
  Installs the necessary tools that are
  needed to run Yetus.
* dev-support/jenkins.sh file.
  Passing of some flags are handled here
  which are needed for the nightly CI.
This commit is contained in:
Gautham B A 2023-05-03 22:44:54 +05:30 committed by GitHub
parent 476f60a806
commit 0d06fd77de
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 133 additions and 43 deletions

View File

@ -20,6 +20,20 @@
# Override these to match Apache Hadoop's requirements # Override these to match Apache Hadoop's requirements
personality_plugins "all,-ant,-gradle,-scalac,-scaladoc" personality_plugins "all,-ant,-gradle,-scalac,-scaladoc"
# These flags are needed to run Yetus against Hadoop on Windows.
WINDOWS_FLAGS="-Pnative-win
-Dhttps.protocols=TLSv1.2
-Drequire.openssl
-Drequire.test.libhadoop
-Dshell-executable=${BASH_EXECUTABLE}
-Dopenssl.prefix=${VCPKG_INSTALLED_PACKAGES}
-Dcmake.prefix.path=${VCPKG_INSTALLED_PACKAGES}
-Dwindows.cmake.toolchain.file=${CMAKE_TOOLCHAIN_FILE}
-Dwindows.cmake.build.type=RelWithDebInfo
-Dwindows.build.hdfspp.dll=off
-Dwindows.no.sasl=on
-Duse.platformToolsetVersion=v142"
## @description Globals specific to this personality ## @description Globals specific to this personality
## @audience private ## @audience private
## @stability evolving ## @stability evolving
@ -87,17 +101,30 @@ function hadoop_order
echo "${hadoopm}" echo "${hadoopm}"
} }
## @description Retrieves the Hadoop project version defined in the root pom.xml
## @audience private
## @stability evolving
## @returns 0 on success, 1 on failure
function load_hadoop_version
{
if [[ -f "${BASEDIR}/pom.xml" ]]; then
HADOOP_VERSION=$(grep '<version>' "${BASEDIR}/pom.xml" \
| head -1 \
| "${SED}" -e 's|^ *<version>||' -e 's|</version>.*$||' \
| cut -f1 -d- )
return 0
else
return 1
fi
}
## @description Determine if it is safe to run parallel tests ## @description Determine if it is safe to run parallel tests
## @audience private ## @audience private
## @stability evolving ## @stability evolving
## @param ordering ## @param ordering
function hadoop_test_parallel function hadoop_test_parallel
{ {
if [[ -f "${BASEDIR}/pom.xml" ]]; then if load_hadoop_version; then
HADOOP_VERSION=$(grep '<version>' "${BASEDIR}/pom.xml" \
| head -1 \
| "${SED}" -e 's|^ *<version>||' -e 's|</version>.*$||' \
| cut -f1 -d- )
export HADOOP_VERSION export HADOOP_VERSION
else else
return 1 return 1
@ -262,7 +289,10 @@ function hadoop_native_flags
Windows_NT|CYGWIN*|MINGW*|MSYS*) Windows_NT|CYGWIN*|MINGW*|MSYS*)
echo \ echo \
"${args[@]}" \ "${args[@]}" \
-Drequire.snappy -Drequire.openssl -Pnative-win -Drequire.snappy \
-Pdist \
-Dtar \
"${WINDOWS_FLAGS}"
;; ;;
*) *)
echo \ echo \
@ -405,7 +435,10 @@ function personality_modules
extra="${extra} ${flags}" extra="${extra} ${flags}"
fi fi
extra="-Ptest-patch ${extra}" if [[ "$IS_WINDOWS" && "$IS_WINDOWS" == 1 ]]; then
extra="-Ptest-patch -Pdist -Dtar ${WINDOWS_FLAGS} ${extra}"
fi
for module in $(hadoop_order ${ordering}); do for module in $(hadoop_order ${ordering}); do
# shellcheck disable=SC2086 # shellcheck disable=SC2086
personality_enqueue_module ${module} ${extra} personality_enqueue_module ${module} ${extra}
@ -548,17 +581,28 @@ function shadedclient_rebuild
big_console_header "Checking client artifacts on ${repostatus} with shaded clients" big_console_header "Checking client artifacts on ${repostatus} with shaded clients"
extra="-Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotbugs.skip=true"
if [[ "$IS_WINDOWS" && "$IS_WINDOWS" == 1 ]]; then
if load_hadoop_version; then
export HADOOP_HOME="${SOURCEDIR}/hadoop-dist/target/hadoop-${HADOOP_VERSION}-SNAPSHOT"
else
yetus_error "[WARNING] Unable to extract the Hadoop version and thus HADOOP_HOME is not set. Some tests may fail."
fi
extra="${WINDOWS_FLAGS} ${extra}"
fi
echo_and_redirect "${logfile}" \ echo_and_redirect "${logfile}" \
"${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am \ "${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am "${modules[@]}" "${extra}"
"${modules[@]}" \
-Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotbugs.skip=true
big_console_header "Checking client artifacts on ${repostatus} with non-shaded clients" big_console_header "Checking client artifacts on ${repostatus} with non-shaded clients"
echo_and_redirect "${logfile}" \ echo_and_redirect "${logfile}" \
"${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am \ "${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am \
"${modules[@]}" \ "${modules[@]}" \
-DskipShade -Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotbugs.skip=true -DskipShade -Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true \
-Dspotbugs.skip=true "${extra}"
count=$("${GREP}" -c '\[ERROR\]' "${logfile}") count=$("${GREP}" -c '\[ERROR\]' "${logfile}")
if [[ ${count} -gt 0 ]]; then if [[ ${count} -gt 0 ]]; then

View File

@ -64,7 +64,7 @@ ENV JAVA_HOME "C:\Java\zulu8.62.0.19-ca-jdk8.0.332-win_x64"
RUN setx PATH "%PATH%;%JAVA_HOME%\bin" RUN setx PATH "%PATH%;%JAVA_HOME%\bin"
# Install Apache Maven. # Install Apache Maven.
RUN powershell Invoke-WebRequest -URI https://dlcdn.apache.org/maven/maven-3/3.8.6/binaries/apache-maven-3.8.6-bin.zip -OutFile $Env:TEMP\apache-maven-3.8.6-bin.zip RUN powershell Invoke-WebRequest -URI https://archive.apache.org/dist/maven/maven-3/3.8.6/binaries/apache-maven-3.8.6-bin.zip -OutFile $Env:TEMP\apache-maven-3.8.6-bin.zip
RUN powershell Expand-Archive -Path $Env:TEMP\apache-maven-3.8.6-bin.zip -DestinationPath "C:\Maven" RUN powershell Expand-Archive -Path $Env:TEMP\apache-maven-3.8.6-bin.zip -DestinationPath "C:\Maven"
RUN setx PATH "%PATH%;C:\Maven\apache-maven-3.8.6\bin" RUN setx PATH "%PATH%;C:\Maven\apache-maven-3.8.6\bin"
ENV MAVEN_OPTS '-Xmx2048M -Xss128M' ENV MAVEN_OPTS '-Xmx2048M -Xss128M'
@ -74,8 +74,45 @@ RUN powershell Invoke-WebRequest -URI https://cmake.org/files/v3.19/cmake-3.19.0
RUN powershell Expand-Archive -Path $Env:TEMP\cmake-3.19.0-win64-x64.zip -DestinationPath "C:\CMake" RUN powershell Expand-Archive -Path $Env:TEMP\cmake-3.19.0-win64-x64.zip -DestinationPath "C:\CMake"
RUN setx PATH "%PATH%;C:\CMake\cmake-3.19.0-win64-x64\bin" RUN setx PATH "%PATH%;C:\CMake\cmake-3.19.0-win64-x64\bin"
# Install zstd 1.5.4.
RUN powershell Invoke-WebRequest -Uri https://github.com/facebook/zstd/releases/download/v1.5.4/zstd-v1.5.4-win64.zip -OutFile $Env:TEMP\zstd-v1.5.4-win64.zip
RUN powershell Expand-Archive -Path $Env:TEMP\zstd-v1.5.4-win64.zip -DestinationPath "C:\ZStd"
RUN setx PATH "%PATH%;C:\ZStd"
# Install libopenssl 3.1.0 needed for rsync 3.2.7.
RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libopenssl-3.1.0-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libopenssl-3.1.0-1-x86_64.pkg.tar.zst
RUN powershell zstd -d $Env:TEMP\libopenssl-3.1.0-1-x86_64.pkg.tar.zst -o $Env:TEMP\libopenssl-3.1.0-1-x86_64.pkg.tar
RUN powershell mkdir "C:\LibOpenSSL"
RUN powershell tar -xvf $Env:TEMP\libopenssl-3.1.0-1-x86_64.pkg.tar -C "C:\LibOpenSSL"
# Install libxxhash 0.8.1 needed for rsync 3.2.7.
RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libxxhash-0.8.1-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar.zst
RUN powershell zstd -d $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar.zst -o $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar
RUN powershell mkdir "C:\LibXXHash"
RUN powershell tar -xvf $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar -C "C:\LibXXHash"
# Install libzstd 1.5.4 needed for rsync 3.2.7.
RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libzstd-1.5.4-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libzstd-1.5.4-1-x86_64.pkg.tar.zst
RUN powershell zstd -d $Env:TEMP\libzstd-1.5.4-1-x86_64.pkg.tar.zst -o $Env:TEMP\libzstd-1.5.4-1-x86_64.pkg.tar
RUN powershell mkdir "C:\LibZStd"
RUN powershell tar -xvf $Env:TEMP\libzstd-1.5.4-1-x86_64.pkg.tar -C "C:\LibZStd"
# Install rsync 3.2.7.
RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/rsync-3.2.7-2-x86_64.pkg.tar.zst -OutFile $Env:TEMP\rsync-3.2.7-2-x86_64.pkg.tar.zst
RUN powershell zstd -d $Env:TEMP\rsync-3.2.7-2-x86_64.pkg.tar.zst -o $Env:TEMP\rsync-3.2.7-2-x86_64.pkg.tar
RUN powershell mkdir "C:\RSync"
RUN powershell tar -xvf $Env:TEMP\rsync-3.2.7-2-x86_64.pkg.tar -C "C:\RSync"
# Copy the dependencies of rsync 3.2.7.
RUN powershell Copy-Item -Path "C:\LibOpenSSL\usr\bin\*.dll" -Destination "C:\Program` Files\Git\usr\bin"
RUN powershell Copy-Item -Path "C:\LibXXHash\usr\bin\*.dll" -Destination "C:\Program` Files\Git\usr\bin"
RUN powershell Copy-Item -Path "C:\LibZStd\usr\bin\*.dll" -Destination "C:\Program` Files\Git\usr\bin"
RUN powershell Copy-Item -Path "C:\RSync\usr\bin\*" -Destination "C:\Program` Files\Git\usr\bin"
# We get strange Javadoc errors without this. # We get strange Javadoc errors without this.
RUN setx classpath "" RUN setx classpath ""
RUN git config --global core.longpaths true
RUN setx PATH "%PATH%;C:\Program Files\Git\usr\bin"
# Define the entry point for the docker container. # Define the entry point for the docker container.
ENTRYPOINT ["C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\BuildTools\\VC\\Auxiliary\\Build\\vcvars64.bat", "&&", "cmd.exe"] ENTRYPOINT ["C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\BuildTools\\VC\\Auxiliary\\Build\\vcvars64.bat", "&&", "cmd.exe"]

View File

@ -114,22 +114,47 @@ function check_ci_run() {
function run_ci() { function run_ci() {
TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/test-patch.sh" TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/test-patch.sh"
# this must be clean for every run if [[ "$IS_WINDOWS" && "$IS_WINDOWS" == 1 ]]; then
if [[ -d "${PATCHDIR}" ]]; then echo "Building in a Windows environment, skipping some Yetus related settings"
rm -rf "${PATCHDIR:?}" else
fi # run in docker mode and specifically point to our
mkdir -p "${PATCHDIR}" # Dockerfile since we don't want to use the auto-pulled version.
YETUS_ARGS+=("--docker")
YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
YETUS_ARGS+=("--mvn-custom-repos")
YETUS_ARGS+=("--dockermemlimit=22g")
# if given a JIRA issue, process it. If CHANGE_URL is set # test with Java 8 and 11
# (e.g., Github Branch Source plugin), process it. YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-8-openjdk-amd64")
# otherwise exit, because we don't want Hadoop to do a YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-11-openjdk-amd64")
# full build. We wouldn't normally do this check for smaller YETUS_ARGS+=("--multijdktests=compile")
# projects. :) fi
if [[ -n "${JIRA_ISSUE_KEY}" ]]; then
YETUS_ARGS+=("${JIRA_ISSUE_KEY}") if [[ "$IS_NIGHTLY_BUILD" && "$IS_NIGHTLY_BUILD" == 1 ]]; then
elif [[ -z "${CHANGE_URL}" ]]; then YETUS_ARGS+=("--empty-patch")
echo "Full build skipped" >"${PATCHDIR}/report.html" YETUS_ARGS+=("--branch=${BRANCH_NAME}")
exit 0 else
# this must be clean for every run
if [[ -d "${PATCHDIR}" ]]; then
rm -rf "${PATCHDIR:?}"
fi
mkdir -p "${PATCHDIR}"
# if given a JIRA issue, process it. If CHANGE_URL is set
# (e.g., Github Branch Source plugin), process it.
# otherwise exit, because we don't want Hadoop to do a
# full build. We wouldn't normally do this check for smaller
# projects. :)
if [[ -n "${JIRA_ISSUE_KEY}" ]]; then
YETUS_ARGS+=("${JIRA_ISSUE_KEY}")
elif [[ -z "${CHANGE_URL}" ]]; then
echo "Full build skipped" >"${PATCHDIR}/report.html"
exit 0
fi
# write Yetus report as GitHub comment (YETUS-1102)
YETUS_ARGS+=("--github-write-comment")
YETUS_ARGS+=("--github-use-emoji-vote")
fi fi
YETUS_ARGS+=("--patch-dir=${PATCHDIR}") YETUS_ARGS+=("--patch-dir=${PATCHDIR}")
@ -156,7 +181,6 @@ function run_ci() {
# changing these to higher values may cause problems # changing these to higher values may cause problems
# with other jobs on systemd-enabled machines # with other jobs on systemd-enabled machines
YETUS_ARGS+=("--proclimit=5500") YETUS_ARGS+=("--proclimit=5500")
YETUS_ARGS+=("--dockermemlimit=22g")
# -1 spotbugs issues that show up prior to the patch being applied # -1 spotbugs issues that show up prior to the patch being applied
YETUS_ARGS+=("--spotbugs-strict-precheck") YETUS_ARGS+=("--spotbugs-strict-precheck")
@ -175,30 +199,15 @@ function run_ci() {
# much attention to them # much attention to them
YETUS_ARGS+=("--tests-filter=checkstyle") YETUS_ARGS+=("--tests-filter=checkstyle")
# run in docker mode and specifically point to our
# Dockerfile since we don't want to use the auto-pulled version.
YETUS_ARGS+=("--docker")
YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
YETUS_ARGS+=("--mvn-custom-repos")
# effectively treat dev-suport as a custom maven module # effectively treat dev-suport as a custom maven module
YETUS_ARGS+=("--skip-dirs=dev-support") YETUS_ARGS+=("--skip-dirs=dev-support")
# help keep the ASF boxes clean # help keep the ASF boxes clean
YETUS_ARGS+=("--sentinel") YETUS_ARGS+=("--sentinel")
# test with Java 8 and 11
YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-8-openjdk-amd64")
YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-11-openjdk-amd64")
YETUS_ARGS+=("--multijdktests=compile")
# custom javadoc goals # custom javadoc goals
YETUS_ARGS+=("--mvn-javadoc-goals=process-sources,javadoc:javadoc-no-fork") YETUS_ARGS+=("--mvn-javadoc-goals=process-sources,javadoc:javadoc-no-fork")
# write Yetus report as GitHub comment (YETUS-1102)
YETUS_ARGS+=("--github-write-comment")
YETUS_ARGS+=("--github-use-emoji-vote")
"${TESTPATCHBIN}" "${YETUS_ARGS[@]}" "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
} }