HADOOP-18134. Setup Jenkins nightly CI for Windows 10 (#5062)
This PR gets Yetus to run on Windows 10 against the Hadoop codebase. It introduces the following changes to allow us to setup the nightly CI on Jenkins for Hadoop on Windows 10. * Hadoop personality changes for Yetus. Additional arguments have been passed, which are necessary to build and test Hadoop on Windows 10. * Docker image for building Hadoop on Windows 10. Installs the necessary tools that are needed to run Yetus. * dev-support/jenkins.sh file. Passing of some flags are handled here which are needed for the nightly CI.
This commit is contained in:
Gautham B A
GitHub
parent
476f60a806
commit
0d06fd77de
@ -20,6 +20,20 @@
|
||||
# Override these to match Apache Hadoop's requirements
|
||||
personality_plugins "all,-ant,-gradle,-scalac,-scaladoc"
|
||||
|
||||
# These flags are needed to run Yetus against Hadoop on Windows.
|
||||
WINDOWS_FLAGS="-Pnative-win
|
||||
-Dhttps.protocols=TLSv1.2
|
||||
-Drequire.openssl
|
||||
-Drequire.test.libhadoop
|
||||
-Dshell-executable=${BASH_EXECUTABLE}
|
||||
-Dopenssl.prefix=${VCPKG_INSTALLED_PACKAGES}
|
||||
-Dcmake.prefix.path=${VCPKG_INSTALLED_PACKAGES}
|
||||
-Dwindows.cmake.toolchain.file=${CMAKE_TOOLCHAIN_FILE}
|
||||
-Dwindows.cmake.build.type=RelWithDebInfo
|
||||
-Dwindows.build.hdfspp.dll=off
|
||||
-Dwindows.no.sasl=on
|
||||
-Duse.platformToolsetVersion=v142"
|
||||
|
||||
## @description Globals specific to this personality
|
||||
## @audience private
|
||||
## @stability evolving
|
||||
@ -87,17 +101,30 @@ function hadoop_order
|
||||
echo "${hadoopm}"
|
||||
}
|
||||
|
||||
## @description Retrieves the Hadoop project version defined in the root pom.xml
|
||||
## @audience private
|
||||
## @stability evolving
|
||||
## @returns 0 on success, 1 on failure
|
||||
function load_hadoop_version
|
||||
{
|
||||
if [[ -f "${BASEDIR}/pom.xml" ]]; then
|
||||
HADOOP_VERSION=$(grep '<version>' "${BASEDIR}/pom.xml" \
|
||||
| head -1 \
|
||||
| "${SED}" -e 's|^ *<version>||' -e 's|</version>.*$||' \
|
||||
| cut -f1 -d- )
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
## @description Determine if it is safe to run parallel tests
|
||||
## @audience private
|
||||
## @stability evolving
|
||||
## @param ordering
|
||||
function hadoop_test_parallel
|
||||
{
|
||||
if [[ -f "${BASEDIR}/pom.xml" ]]; then
|
||||
HADOOP_VERSION=$(grep '<version>' "${BASEDIR}/pom.xml" \
|
||||
| head -1 \
|
||||
| "${SED}" -e 's|^ *<version>||' -e 's|</version>.*$||' \
|
||||
| cut -f1 -d- )
|
||||
if load_hadoop_version; then
|
||||
export HADOOP_VERSION
|
||||
else
|
||||
return 1
|
||||
@ -262,7 +289,10 @@ function hadoop_native_flags
|
||||
Windows_NT|CYGWIN*|MINGW*|MSYS*)
|
||||
echo \
|
||||
"${args[@]}" \
|
||||
-Drequire.snappy -Drequire.openssl -Pnative-win
|
||||
-Drequire.snappy \
|
||||
-Pdist \
|
||||
-Dtar \
|
||||
"${WINDOWS_FLAGS}"
|
||||
;;
|
||||
*)
|
||||
echo \
|
||||
@ -405,7 +435,10 @@ function personality_modules
|
||||
extra="${extra} ${flags}"
|
||||
fi
|
||||
|
||||
extra="-Ptest-patch ${extra}"
|
||||
if [[ "$IS_WINDOWS" && "$IS_WINDOWS" == 1 ]]; then
|
||||
extra="-Ptest-patch -Pdist -Dtar ${WINDOWS_FLAGS} ${extra}"
|
||||
fi
|
||||
|
||||
for module in $(hadoop_order ${ordering}); do
|
||||
# shellcheck disable=SC2086
|
||||
personality_enqueue_module ${module} ${extra}
|
||||
@ -548,17 +581,28 @@ function shadedclient_rebuild
|
||||
|
||||
big_console_header "Checking client artifacts on ${repostatus} with shaded clients"
|
||||
|
||||
extra="-Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotbugs.skip=true"
|
||||
|
||||
if [[ "$IS_WINDOWS" && "$IS_WINDOWS" == 1 ]]; then
|
||||
if load_hadoop_version; then
|
||||
export HADOOP_HOME="${SOURCEDIR}/hadoop-dist/target/hadoop-${HADOOP_VERSION}-SNAPSHOT"
|
||||
else
|
||||
yetus_error "[WARNING] Unable to extract the Hadoop version and thus HADOOP_HOME is not set. Some tests may fail."
|
||||
fi
|
||||
|
||||
extra="${WINDOWS_FLAGS} ${extra}"
|
||||
fi
|
||||
|
||||
echo_and_redirect "${logfile}" \
|
||||
"${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am \
|
||||
"${modules[@]}" \
|
||||
-Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotbugs.skip=true
|
||||
"${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am "${modules[@]}" "${extra}"
|
||||
|
||||
big_console_header "Checking client artifacts on ${repostatus} with non-shaded clients"
|
||||
|
||||
echo_and_redirect "${logfile}" \
|
||||
"${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am \
|
||||
"${modules[@]}" \
|
||||
-DskipShade -Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotbugs.skip=true
|
||||
-DskipShade -Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true \
|
||||
-Dspotbugs.skip=true "${extra}"
|
||||
|
||||
count=$("${GREP}" -c '\[ERROR\]' "${logfile}")
|
||||
if [[ ${count} -gt 0 ]]; then
|
||||
|
||||
@ -64,7 +64,7 @@ ENV JAVA_HOME "C:\Java\zulu8.62.0.19-ca-jdk8.0.332-win_x64"
|
||||
RUN setx PATH "%PATH%;%JAVA_HOME%\bin"
|
||||
|
||||
# Install Apache Maven.
|
||||
RUN powershell Invoke-WebRequest -URI https://dlcdn.apache.org/maven/maven-3/3.8.6/binaries/apache-maven-3.8.6-bin.zip -OutFile $Env:TEMP\apache-maven-3.8.6-bin.zip
|
||||
RUN powershell Invoke-WebRequest -URI https://archive.apache.org/dist/maven/maven-3/3.8.6/binaries/apache-maven-3.8.6-bin.zip -OutFile $Env:TEMP\apache-maven-3.8.6-bin.zip
|
||||
RUN powershell Expand-Archive -Path $Env:TEMP\apache-maven-3.8.6-bin.zip -DestinationPath "C:\Maven"
|
||||
RUN setx PATH "%PATH%;C:\Maven\apache-maven-3.8.6\bin"
|
||||
ENV MAVEN_OPTS '-Xmx2048M -Xss128M'
|
||||
@ -74,8 +74,45 @@ RUN powershell Invoke-WebRequest -URI https://cmake.org/files/v3.19/cmake-3.19.0
|
||||
RUN powershell Expand-Archive -Path $Env:TEMP\cmake-3.19.0-win64-x64.zip -DestinationPath "C:\CMake"
|
||||
RUN setx PATH "%PATH%;C:\CMake\cmake-3.19.0-win64-x64\bin"
|
||||
|
||||
# Install zstd 1.5.4.
|
||||
RUN powershell Invoke-WebRequest -Uri https://github.com/facebook/zstd/releases/download/v1.5.4/zstd-v1.5.4-win64.zip -OutFile $Env:TEMP\zstd-v1.5.4-win64.zip
|
||||
RUN powershell Expand-Archive -Path $Env:TEMP\zstd-v1.5.4-win64.zip -DestinationPath "C:\ZStd"
|
||||
RUN setx PATH "%PATH%;C:\ZStd"
|
||||
|
||||
# Install libopenssl 3.1.0 needed for rsync 3.2.7.
|
||||
RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libopenssl-3.1.0-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libopenssl-3.1.0-1-x86_64.pkg.tar.zst
|
||||
RUN powershell zstd -d $Env:TEMP\libopenssl-3.1.0-1-x86_64.pkg.tar.zst -o $Env:TEMP\libopenssl-3.1.0-1-x86_64.pkg.tar
|
||||
RUN powershell mkdir "C:\LibOpenSSL"
|
||||
RUN powershell tar -xvf $Env:TEMP\libopenssl-3.1.0-1-x86_64.pkg.tar -C "C:\LibOpenSSL"
|
||||
|
||||
# Install libxxhash 0.8.1 needed for rsync 3.2.7.
|
||||
RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libxxhash-0.8.1-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar.zst
|
||||
RUN powershell zstd -d $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar.zst -o $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar
|
||||
RUN powershell mkdir "C:\LibXXHash"
|
||||
RUN powershell tar -xvf $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar -C "C:\LibXXHash"
|
||||
|
||||
# Install libzstd 1.5.4 needed for rsync 3.2.7.
|
||||
RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libzstd-1.5.4-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libzstd-1.5.4-1-x86_64.pkg.tar.zst
|
||||
RUN powershell zstd -d $Env:TEMP\libzstd-1.5.4-1-x86_64.pkg.tar.zst -o $Env:TEMP\libzstd-1.5.4-1-x86_64.pkg.tar
|
||||
RUN powershell mkdir "C:\LibZStd"
|
||||
RUN powershell tar -xvf $Env:TEMP\libzstd-1.5.4-1-x86_64.pkg.tar -C "C:\LibZStd"
|
||||
|
||||
# Install rsync 3.2.7.
|
||||
RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/rsync-3.2.7-2-x86_64.pkg.tar.zst -OutFile $Env:TEMP\rsync-3.2.7-2-x86_64.pkg.tar.zst
|
||||
RUN powershell zstd -d $Env:TEMP\rsync-3.2.7-2-x86_64.pkg.tar.zst -o $Env:TEMP\rsync-3.2.7-2-x86_64.pkg.tar
|
||||
RUN powershell mkdir "C:\RSync"
|
||||
RUN powershell tar -xvf $Env:TEMP\rsync-3.2.7-2-x86_64.pkg.tar -C "C:\RSync"
|
||||
# Copy the dependencies of rsync 3.2.7.
|
||||
RUN powershell Copy-Item -Path "C:\LibOpenSSL\usr\bin\*.dll" -Destination "C:\Program` Files\Git\usr\bin"
|
||||
RUN powershell Copy-Item -Path "C:\LibXXHash\usr\bin\*.dll" -Destination "C:\Program` Files\Git\usr\bin"
|
||||
RUN powershell Copy-Item -Path "C:\LibZStd\usr\bin\*.dll" -Destination "C:\Program` Files\Git\usr\bin"
|
||||
RUN powershell Copy-Item -Path "C:\RSync\usr\bin\*" -Destination "C:\Program` Files\Git\usr\bin"
|
||||
|
||||
# We get strange Javadoc errors without this.
|
||||
RUN setx classpath ""
|
||||
|
||||
RUN git config --global core.longpaths true
|
||||
RUN setx PATH "%PATH%;C:\Program Files\Git\usr\bin"
|
||||
|
||||
# Define the entry point for the docker container.
|
||||
ENTRYPOINT ["C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\BuildTools\\VC\\Auxiliary\\Build\\vcvars64.bat", "&&", "cmd.exe"]
|
||||
|
||||
@ -114,22 +114,47 @@ function check_ci_run() {
|
||||
function run_ci() {
|
||||
TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/test-patch.sh"
|
||||
|
||||
# this must be clean for every run
|
||||
if [[ -d "${PATCHDIR}" ]]; then
|
||||
rm -rf "${PATCHDIR:?}"
|
||||
fi
|
||||
mkdir -p "${PATCHDIR}"
|
||||
if [[ "$IS_WINDOWS" && "$IS_WINDOWS" == 1 ]]; then
|
||||
echo "Building in a Windows environment, skipping some Yetus related settings"
|
||||
else
|
||||
# run in docker mode and specifically point to our
|
||||
# Dockerfile since we don't want to use the auto-pulled version.
|
||||
YETUS_ARGS+=("--docker")
|
||||
YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
|
||||
YETUS_ARGS+=("--mvn-custom-repos")
|
||||
YETUS_ARGS+=("--dockermemlimit=22g")
|
||||
|
||||
# if given a JIRA issue, process it. If CHANGE_URL is set
|
||||
# (e.g., Github Branch Source plugin), process it.
|
||||
# otherwise exit, because we don't want Hadoop to do a
|
||||
# full build. We wouldn't normally do this check for smaller
|
||||
# projects. :)
|
||||
if [[ -n "${JIRA_ISSUE_KEY}" ]]; then
|
||||
YETUS_ARGS+=("${JIRA_ISSUE_KEY}")
|
||||
elif [[ -z "${CHANGE_URL}" ]]; then
|
||||
echo "Full build skipped" >"${PATCHDIR}/report.html"
|
||||
exit 0
|
||||
# test with Java 8 and 11
|
||||
YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-8-openjdk-amd64")
|
||||
YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-11-openjdk-amd64")
|
||||
YETUS_ARGS+=("--multijdktests=compile")
|
||||
fi
|
||||
|
||||
if [[ "$IS_NIGHTLY_BUILD" && "$IS_NIGHTLY_BUILD" == 1 ]]; then
|
||||
YETUS_ARGS+=("--empty-patch")
|
||||
YETUS_ARGS+=("--branch=${BRANCH_NAME}")
|
||||
else
|
||||
# this must be clean for every run
|
||||
if [[ -d "${PATCHDIR}" ]]; then
|
||||
rm -rf "${PATCHDIR:?}"
|
||||
fi
|
||||
mkdir -p "${PATCHDIR}"
|
||||
|
||||
# if given a JIRA issue, process it. If CHANGE_URL is set
|
||||
# (e.g., Github Branch Source plugin), process it.
|
||||
# otherwise exit, because we don't want Hadoop to do a
|
||||
# full build. We wouldn't normally do this check for smaller
|
||||
# projects. :)
|
||||
if [[ -n "${JIRA_ISSUE_KEY}" ]]; then
|
||||
YETUS_ARGS+=("${JIRA_ISSUE_KEY}")
|
||||
elif [[ -z "${CHANGE_URL}" ]]; then
|
||||
echo "Full build skipped" >"${PATCHDIR}/report.html"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# write Yetus report as GitHub comment (YETUS-1102)
|
||||
YETUS_ARGS+=("--github-write-comment")
|
||||
YETUS_ARGS+=("--github-use-emoji-vote")
|
||||
fi
|
||||
|
||||
YETUS_ARGS+=("--patch-dir=${PATCHDIR}")
|
||||
@ -156,7 +181,6 @@ function run_ci() {
|
||||
# changing these to higher values may cause problems
|
||||
# with other jobs on systemd-enabled machines
|
||||
YETUS_ARGS+=("--proclimit=5500")
|
||||
YETUS_ARGS+=("--dockermemlimit=22g")
|
||||
|
||||
# -1 spotbugs issues that show up prior to the patch being applied
|
||||
YETUS_ARGS+=("--spotbugs-strict-precheck")
|
||||
@ -175,30 +199,15 @@ function run_ci() {
|
||||
# much attention to them
|
||||
YETUS_ARGS+=("--tests-filter=checkstyle")
|
||||
|
||||
# run in docker mode and specifically point to our
|
||||
# Dockerfile since we don't want to use the auto-pulled version.
|
||||
YETUS_ARGS+=("--docker")
|
||||
YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
|
||||
YETUS_ARGS+=("--mvn-custom-repos")
|
||||
|
||||
# effectively treat dev-suport as a custom maven module
|
||||
YETUS_ARGS+=("--skip-dirs=dev-support")
|
||||
|
||||
# help keep the ASF boxes clean
|
||||
YETUS_ARGS+=("--sentinel")
|
||||
|
||||
# test with Java 8 and 11
|
||||
YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-8-openjdk-amd64")
|
||||
YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-11-openjdk-amd64")
|
||||
YETUS_ARGS+=("--multijdktests=compile")
|
||||
|
||||
# custom javadoc goals
|
||||
YETUS_ARGS+=("--mvn-javadoc-goals=process-sources,javadoc:javadoc-no-fork")
|
||||
|
||||
# write Yetus report as GitHub comment (YETUS-1102)
|
||||
YETUS_ARGS+=("--github-write-comment")
|
||||
YETUS_ARGS+=("--github-use-emoji-vote")
|
||||
|
||||
"${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user