diff --git a/dev-support/bin/hadoop.sh b/dev-support/bin/hadoop.sh index 763b0507e4..b51a9525bd 100755 --- a/dev-support/bin/hadoop.sh +++ b/dev-support/bin/hadoop.sh @@ -20,6 +20,20 @@ # Override these to match Apache Hadoop's requirements personality_plugins "all,-ant,-gradle,-scalac,-scaladoc" +# These flags are needed to run Yetus against Hadoop on Windows. +WINDOWS_FLAGS="-Pnative-win + -Dhttps.protocols=TLSv1.2 + -Drequire.openssl + -Drequire.test.libhadoop + -Dshell-executable=${BASH_EXECUTABLE} + -Dopenssl.prefix=${VCPKG_INSTALLED_PACKAGES} + -Dcmake.prefix.path=${VCPKG_INSTALLED_PACKAGES} + -Dwindows.cmake.toolchain.file=${CMAKE_TOOLCHAIN_FILE} + -Dwindows.cmake.build.type=RelWithDebInfo + -Dwindows.build.hdfspp.dll=off + -Dwindows.no.sasl=on + -Duse.platformToolsetVersion=v142" + ## @description Globals specific to this personality ## @audience private ## @stability evolving @@ -87,17 +101,30 @@ function hadoop_order echo "${hadoopm}" } +## @description Retrieves the Hadoop project version defined in the root pom.xml +## @audience private +## @stability evolving +## @returns 0 on success, 1 on failure +function load_hadoop_version +{ + if [[ -f "${BASEDIR}/pom.xml" ]]; then + HADOOP_VERSION=$(grep '' "${BASEDIR}/pom.xml" \ + | head -1 \ + | "${SED}" -e 's|^ *||' -e 's|.*$||' \ + | cut -f1 -d- ) + return 0 + else + return 1 + fi +} + ## @description Determine if it is safe to run parallel tests ## @audience private ## @stability evolving ## @param ordering function hadoop_test_parallel { - if [[ -f "${BASEDIR}/pom.xml" ]]; then - HADOOP_VERSION=$(grep '' "${BASEDIR}/pom.xml" \ - | head -1 \ - | "${SED}" -e 's|^ *||' -e 's|.*$||' \ - | cut -f1 -d- ) + if load_hadoop_version; then export HADOOP_VERSION else return 1 @@ -262,7 +289,10 @@ function hadoop_native_flags Windows_NT|CYGWIN*|MINGW*|MSYS*) echo \ "${args[@]}" \ - -Drequire.snappy -Drequire.openssl -Pnative-win + -Drequire.snappy \ + -Pdist \ + -Dtar \ + "${WINDOWS_FLAGS}" ;; *) echo \ @@ -405,7 +435,10 @@ function personality_modules extra="${extra} ${flags}" fi - extra="-Ptest-patch ${extra}" + if [[ "$IS_WINDOWS" && "$IS_WINDOWS" == 1 ]]; then + extra="-Ptest-patch -Pdist -Dtar ${WINDOWS_FLAGS} ${extra}" + fi + for module in $(hadoop_order ${ordering}); do # shellcheck disable=SC2086 personality_enqueue_module ${module} ${extra} @@ -548,17 +581,28 @@ function shadedclient_rebuild big_console_header "Checking client artifacts on ${repostatus} with shaded clients" + extra="-Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotbugs.skip=true" + + if [[ "$IS_WINDOWS" && "$IS_WINDOWS" == 1 ]]; then + if load_hadoop_version; then + export HADOOP_HOME="${SOURCEDIR}/hadoop-dist/target/hadoop-${HADOOP_VERSION}-SNAPSHOT" + else + yetus_error "[WARNING] Unable to extract the Hadoop version and thus HADOOP_HOME is not set. Some tests may fail." + fi + + extra="${WINDOWS_FLAGS} ${extra}" + fi + echo_and_redirect "${logfile}" \ - "${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am \ - "${modules[@]}" \ - -Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotbugs.skip=true + "${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am "${modules[@]}" "${extra}" big_console_header "Checking client artifacts on ${repostatus} with non-shaded clients" echo_and_redirect "${logfile}" \ "${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am \ "${modules[@]}" \ - -DskipShade -Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotbugs.skip=true + -DskipShade -Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true \ + -Dspotbugs.skip=true "${extra}" count=$("${GREP}" -c '\[ERROR\]' "${logfile}") if [[ ${count} -gt 0 ]]; then diff --git a/dev-support/docker/Dockerfile_windows_10 b/dev-support/docker/Dockerfile_windows_10 index 7a69a2727a..34ac8ca198 100644 --- a/dev-support/docker/Dockerfile_windows_10 +++ b/dev-support/docker/Dockerfile_windows_10 @@ -64,7 +64,7 @@ ENV JAVA_HOME "C:\Java\zulu8.62.0.19-ca-jdk8.0.332-win_x64" RUN setx PATH "%PATH%;%JAVA_HOME%\bin" # Install Apache Maven. -RUN powershell Invoke-WebRequest -URI https://dlcdn.apache.org/maven/maven-3/3.8.6/binaries/apache-maven-3.8.6-bin.zip -OutFile $Env:TEMP\apache-maven-3.8.6-bin.zip +RUN powershell Invoke-WebRequest -URI https://archive.apache.org/dist/maven/maven-3/3.8.6/binaries/apache-maven-3.8.6-bin.zip -OutFile $Env:TEMP\apache-maven-3.8.6-bin.zip RUN powershell Expand-Archive -Path $Env:TEMP\apache-maven-3.8.6-bin.zip -DestinationPath "C:\Maven" RUN setx PATH "%PATH%;C:\Maven\apache-maven-3.8.6\bin" ENV MAVEN_OPTS '-Xmx2048M -Xss128M' @@ -74,8 +74,45 @@ RUN powershell Invoke-WebRequest -URI https://cmake.org/files/v3.19/cmake-3.19.0 RUN powershell Expand-Archive -Path $Env:TEMP\cmake-3.19.0-win64-x64.zip -DestinationPath "C:\CMake" RUN setx PATH "%PATH%;C:\CMake\cmake-3.19.0-win64-x64\bin" +# Install zstd 1.5.4. +RUN powershell Invoke-WebRequest -Uri https://github.com/facebook/zstd/releases/download/v1.5.4/zstd-v1.5.4-win64.zip -OutFile $Env:TEMP\zstd-v1.5.4-win64.zip +RUN powershell Expand-Archive -Path $Env:TEMP\zstd-v1.5.4-win64.zip -DestinationPath "C:\ZStd" +RUN setx PATH "%PATH%;C:\ZStd" + +# Install libopenssl 3.1.0 needed for rsync 3.2.7. +RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libopenssl-3.1.0-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libopenssl-3.1.0-1-x86_64.pkg.tar.zst +RUN powershell zstd -d $Env:TEMP\libopenssl-3.1.0-1-x86_64.pkg.tar.zst -o $Env:TEMP\libopenssl-3.1.0-1-x86_64.pkg.tar +RUN powershell mkdir "C:\LibOpenSSL" +RUN powershell tar -xvf $Env:TEMP\libopenssl-3.1.0-1-x86_64.pkg.tar -C "C:\LibOpenSSL" + +# Install libxxhash 0.8.1 needed for rsync 3.2.7. +RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libxxhash-0.8.1-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar.zst +RUN powershell zstd -d $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar.zst -o $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar +RUN powershell mkdir "C:\LibXXHash" +RUN powershell tar -xvf $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar -C "C:\LibXXHash" + +# Install libzstd 1.5.4 needed for rsync 3.2.7. +RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libzstd-1.5.4-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libzstd-1.5.4-1-x86_64.pkg.tar.zst +RUN powershell zstd -d $Env:TEMP\libzstd-1.5.4-1-x86_64.pkg.tar.zst -o $Env:TEMP\libzstd-1.5.4-1-x86_64.pkg.tar +RUN powershell mkdir "C:\LibZStd" +RUN powershell tar -xvf $Env:TEMP\libzstd-1.5.4-1-x86_64.pkg.tar -C "C:\LibZStd" + +# Install rsync 3.2.7. +RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/rsync-3.2.7-2-x86_64.pkg.tar.zst -OutFile $Env:TEMP\rsync-3.2.7-2-x86_64.pkg.tar.zst +RUN powershell zstd -d $Env:TEMP\rsync-3.2.7-2-x86_64.pkg.tar.zst -o $Env:TEMP\rsync-3.2.7-2-x86_64.pkg.tar +RUN powershell mkdir "C:\RSync" +RUN powershell tar -xvf $Env:TEMP\rsync-3.2.7-2-x86_64.pkg.tar -C "C:\RSync" +# Copy the dependencies of rsync 3.2.7. +RUN powershell Copy-Item -Path "C:\LibOpenSSL\usr\bin\*.dll" -Destination "C:\Program` Files\Git\usr\bin" +RUN powershell Copy-Item -Path "C:\LibXXHash\usr\bin\*.dll" -Destination "C:\Program` Files\Git\usr\bin" +RUN powershell Copy-Item -Path "C:\LibZStd\usr\bin\*.dll" -Destination "C:\Program` Files\Git\usr\bin" +RUN powershell Copy-Item -Path "C:\RSync\usr\bin\*" -Destination "C:\Program` Files\Git\usr\bin" + # We get strange Javadoc errors without this. RUN setx classpath "" +RUN git config --global core.longpaths true +RUN setx PATH "%PATH%;C:\Program Files\Git\usr\bin" + # Define the entry point for the docker container. ENTRYPOINT ["C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\BuildTools\\VC\\Auxiliary\\Build\\vcvars64.bat", "&&", "cmd.exe"] diff --git a/dev-support/jenkins.sh b/dev-support/jenkins.sh index ed6ef184b4..b335a65159 100644 --- a/dev-support/jenkins.sh +++ b/dev-support/jenkins.sh @@ -114,22 +114,47 @@ function check_ci_run() { function run_ci() { TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/test-patch.sh" - # this must be clean for every run - if [[ -d "${PATCHDIR}" ]]; then - rm -rf "${PATCHDIR:?}" - fi - mkdir -p "${PATCHDIR}" + if [[ "$IS_WINDOWS" && "$IS_WINDOWS" == 1 ]]; then + echo "Building in a Windows environment, skipping some Yetus related settings" + else + # run in docker mode and specifically point to our + # Dockerfile since we don't want to use the auto-pulled version. + YETUS_ARGS+=("--docker") + YETUS_ARGS+=("--dockerfile=${DOCKERFILE}") + YETUS_ARGS+=("--mvn-custom-repos") + YETUS_ARGS+=("--dockermemlimit=22g") - # if given a JIRA issue, process it. If CHANGE_URL is set - # (e.g., Github Branch Source plugin), process it. - # otherwise exit, because we don't want Hadoop to do a - # full build. We wouldn't normally do this check for smaller - # projects. :) - if [[ -n "${JIRA_ISSUE_KEY}" ]]; then - YETUS_ARGS+=("${JIRA_ISSUE_KEY}") - elif [[ -z "${CHANGE_URL}" ]]; then - echo "Full build skipped" >"${PATCHDIR}/report.html" - exit 0 + # test with Java 8 and 11 + YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-8-openjdk-amd64") + YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-11-openjdk-amd64") + YETUS_ARGS+=("--multijdktests=compile") + fi + + if [[ "$IS_NIGHTLY_BUILD" && "$IS_NIGHTLY_BUILD" == 1 ]]; then + YETUS_ARGS+=("--empty-patch") + YETUS_ARGS+=("--branch=${BRANCH_NAME}") + else + # this must be clean for every run + if [[ -d "${PATCHDIR}" ]]; then + rm -rf "${PATCHDIR:?}" + fi + mkdir -p "${PATCHDIR}" + + # if given a JIRA issue, process it. If CHANGE_URL is set + # (e.g., Github Branch Source plugin), process it. + # otherwise exit, because we don't want Hadoop to do a + # full build. We wouldn't normally do this check for smaller + # projects. :) + if [[ -n "${JIRA_ISSUE_KEY}" ]]; then + YETUS_ARGS+=("${JIRA_ISSUE_KEY}") + elif [[ -z "${CHANGE_URL}" ]]; then + echo "Full build skipped" >"${PATCHDIR}/report.html" + exit 0 + fi + + # write Yetus report as GitHub comment (YETUS-1102) + YETUS_ARGS+=("--github-write-comment") + YETUS_ARGS+=("--github-use-emoji-vote") fi YETUS_ARGS+=("--patch-dir=${PATCHDIR}") @@ -156,7 +181,6 @@ function run_ci() { # changing these to higher values may cause problems # with other jobs on systemd-enabled machines YETUS_ARGS+=("--proclimit=5500") - YETUS_ARGS+=("--dockermemlimit=22g") # -1 spotbugs issues that show up prior to the patch being applied YETUS_ARGS+=("--spotbugs-strict-precheck") @@ -175,30 +199,15 @@ function run_ci() { # much attention to them YETUS_ARGS+=("--tests-filter=checkstyle") - # run in docker mode and specifically point to our - # Dockerfile since we don't want to use the auto-pulled version. - YETUS_ARGS+=("--docker") - YETUS_ARGS+=("--dockerfile=${DOCKERFILE}") - YETUS_ARGS+=("--mvn-custom-repos") - # effectively treat dev-suport as a custom maven module YETUS_ARGS+=("--skip-dirs=dev-support") # help keep the ASF boxes clean YETUS_ARGS+=("--sentinel") - # test with Java 8 and 11 - YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-8-openjdk-amd64") - YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-11-openjdk-amd64") - YETUS_ARGS+=("--multijdktests=compile") - # custom javadoc goals YETUS_ARGS+=("--mvn-javadoc-goals=process-sources,javadoc:javadoc-no-fork") - # write Yetus report as GitHub comment (YETUS-1102) - YETUS_ARGS+=("--github-write-comment") - YETUS_ARGS+=("--github-use-emoji-vote") - "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" }