0d06fd77de
This PR gets Yetus to run on Windows 10 against the Hadoop codebase. It introduces the following changes to allow us to setup the nightly CI on Jenkins for Hadoop on Windows 10. * Hadoop personality changes for Yetus. Additional arguments have been passed, which are necessary to build and test Hadoop on Windows 10. * Docker image for building Hadoop on Windows 10. Installs the necessary tools that are needed to run Yetus. * dev-support/jenkins.sh file. Passing of some flags are handled here which are needed for the nightly CI.
119 lines
6.5 KiB
Plaintext
119 lines
6.5 KiB
Plaintext
# Licensed to the Apache Software Foundation (ASF) under one
|
|
# or more contributor license agreements. See the NOTICE file
|
|
# distributed with this work for additional information
|
|
# regarding copyright ownership. The ASF licenses this file
|
|
# to you under the Apache License, Version 2.0 (the
|
|
# "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# Dockerfile for installing the necessary dependencies for building Hadoop.
|
|
# See BUILDING.txt.
|
|
|
|
FROM mcr.microsoft.com/windows:ltsc2019
|
|
|
|
# Need to disable the progress bar for speeding up the downloads.
|
|
# hadolint ignore=SC2086
|
|
RUN powershell $Global:ProgressPreference = 'SilentlyContinue'
|
|
|
|
# Restore the default Windows shell for correct batch processing.
|
|
SHELL ["cmd", "/S", "/C"]
|
|
|
|
# Install Visual Studio 2019 Build Tools.
|
|
RUN curl -SL --output vs_buildtools.exe https://aka.ms/vs/16/release/vs_buildtools.exe \
|
|
&& (start /w vs_buildtools.exe --quiet --wait --norestart --nocache \
|
|
--installPath "%ProgramFiles(x86)%\Microsoft Visual Studio\2019\BuildTools" \
|
|
--add Microsoft.VisualStudio.Workload.VCTools \
|
|
--add Microsoft.VisualStudio.Component.VC.ASAN \
|
|
--add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 \
|
|
--add Microsoft.VisualStudio.Component.Windows10SDK.19041 \
|
|
|| IF "%ERRORLEVEL%"=="3010" EXIT 0) \
|
|
&& del /q vs_buildtools.exe
|
|
|
|
# Install Chocolatey.
|
|
RUN powershell -NoProfile -ExecutionPolicy Bypass -Command "iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))"
|
|
RUN setx PATH "%PATH%;%ALLUSERSPROFILE%\chocolatey\bin"
|
|
|
|
# Install git.
|
|
RUN choco install git.install -y
|
|
RUN powershell Copy-Item -Recurse -Path 'C:\Program Files\Git' -Destination C:\Git
|
|
|
|
# Install vcpkg.
|
|
# hadolint ignore=DL3003
|
|
RUN powershell git clone https://github.com/microsoft/vcpkg.git \
|
|
&& cd vcpkg \
|
|
&& git checkout 7ffa425e1db8b0c3edf9c50f2f3a0f25a324541d \
|
|
&& .\bootstrap-vcpkg.bat
|
|
RUN powershell .\vcpkg\vcpkg.exe install boost:x64-windows
|
|
RUN powershell .\vcpkg\vcpkg.exe install protobuf:x64-windows
|
|
RUN powershell .\vcpkg\vcpkg.exe install openssl:x64-windows
|
|
RUN powershell .\vcpkg\vcpkg.exe install zlib:x64-windows
|
|
ENV PROTOBUF_HOME "C:\vcpkg\installed\x64-windows"
|
|
|
|
# Install Azul Java 8 JDK.
|
|
RUN powershell Invoke-WebRequest -URI https://cdn.azul.com/zulu/bin/zulu8.62.0.19-ca-jdk8.0.332-win_x64.zip -OutFile $Env:TEMP\zulu8.62.0.19-ca-jdk8.0.332-win_x64.zip
|
|
RUN powershell Expand-Archive -Path $Env:TEMP\zulu8.62.0.19-ca-jdk8.0.332-win_x64.zip -DestinationPath "C:\Java"
|
|
ENV JAVA_HOME "C:\Java\zulu8.62.0.19-ca-jdk8.0.332-win_x64"
|
|
RUN setx PATH "%PATH%;%JAVA_HOME%\bin"
|
|
|
|
# Install Apache Maven.
|
|
RUN powershell Invoke-WebRequest -URI https://archive.apache.org/dist/maven/maven-3/3.8.6/binaries/apache-maven-3.8.6-bin.zip -OutFile $Env:TEMP\apache-maven-3.8.6-bin.zip
|
|
RUN powershell Expand-Archive -Path $Env:TEMP\apache-maven-3.8.6-bin.zip -DestinationPath "C:\Maven"
|
|
RUN setx PATH "%PATH%;C:\Maven\apache-maven-3.8.6\bin"
|
|
ENV MAVEN_OPTS '-Xmx2048M -Xss128M'
|
|
|
|
# Install CMake 3.19.0.
|
|
RUN powershell Invoke-WebRequest -URI https://cmake.org/files/v3.19/cmake-3.19.0-win64-x64.zip -OutFile $Env:TEMP\cmake-3.19.0-win64-x64.zip
|
|
RUN powershell Expand-Archive -Path $Env:TEMP\cmake-3.19.0-win64-x64.zip -DestinationPath "C:\CMake"
|
|
RUN setx PATH "%PATH%;C:\CMake\cmake-3.19.0-win64-x64\bin"
|
|
|
|
# Install zstd 1.5.4.
|
|
RUN powershell Invoke-WebRequest -Uri https://github.com/facebook/zstd/releases/download/v1.5.4/zstd-v1.5.4-win64.zip -OutFile $Env:TEMP\zstd-v1.5.4-win64.zip
|
|
RUN powershell Expand-Archive -Path $Env:TEMP\zstd-v1.5.4-win64.zip -DestinationPath "C:\ZStd"
|
|
RUN setx PATH "%PATH%;C:\ZStd"
|
|
|
|
# Install libopenssl 3.1.0 needed for rsync 3.2.7.
|
|
RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libopenssl-3.1.0-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libopenssl-3.1.0-1-x86_64.pkg.tar.zst
|
|
RUN powershell zstd -d $Env:TEMP\libopenssl-3.1.0-1-x86_64.pkg.tar.zst -o $Env:TEMP\libopenssl-3.1.0-1-x86_64.pkg.tar
|
|
RUN powershell mkdir "C:\LibOpenSSL"
|
|
RUN powershell tar -xvf $Env:TEMP\libopenssl-3.1.0-1-x86_64.pkg.tar -C "C:\LibOpenSSL"
|
|
|
|
# Install libxxhash 0.8.1 needed for rsync 3.2.7.
|
|
RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libxxhash-0.8.1-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar.zst
|
|
RUN powershell zstd -d $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar.zst -o $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar
|
|
RUN powershell mkdir "C:\LibXXHash"
|
|
RUN powershell tar -xvf $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar -C "C:\LibXXHash"
|
|
|
|
# Install libzstd 1.5.4 needed for rsync 3.2.7.
|
|
RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libzstd-1.5.4-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libzstd-1.5.4-1-x86_64.pkg.tar.zst
|
|
RUN powershell zstd -d $Env:TEMP\libzstd-1.5.4-1-x86_64.pkg.tar.zst -o $Env:TEMP\libzstd-1.5.4-1-x86_64.pkg.tar
|
|
RUN powershell mkdir "C:\LibZStd"
|
|
RUN powershell tar -xvf $Env:TEMP\libzstd-1.5.4-1-x86_64.pkg.tar -C "C:\LibZStd"
|
|
|
|
# Install rsync 3.2.7.
|
|
RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/rsync-3.2.7-2-x86_64.pkg.tar.zst -OutFile $Env:TEMP\rsync-3.2.7-2-x86_64.pkg.tar.zst
|
|
RUN powershell zstd -d $Env:TEMP\rsync-3.2.7-2-x86_64.pkg.tar.zst -o $Env:TEMP\rsync-3.2.7-2-x86_64.pkg.tar
|
|
RUN powershell mkdir "C:\RSync"
|
|
RUN powershell tar -xvf $Env:TEMP\rsync-3.2.7-2-x86_64.pkg.tar -C "C:\RSync"
|
|
# Copy the dependencies of rsync 3.2.7.
|
|
RUN powershell Copy-Item -Path "C:\LibOpenSSL\usr\bin\*.dll" -Destination "C:\Program` Files\Git\usr\bin"
|
|
RUN powershell Copy-Item -Path "C:\LibXXHash\usr\bin\*.dll" -Destination "C:\Program` Files\Git\usr\bin"
|
|
RUN powershell Copy-Item -Path "C:\LibZStd\usr\bin\*.dll" -Destination "C:\Program` Files\Git\usr\bin"
|
|
RUN powershell Copy-Item -Path "C:\RSync\usr\bin\*" -Destination "C:\Program` Files\Git\usr\bin"
|
|
|
|
# We get strange Javadoc errors without this.
|
|
RUN setx classpath ""
|
|
|
|
RUN git config --global core.longpaths true
|
|
RUN setx PATH "%PATH%;C:\Program Files\Git\usr\bin"
|
|
|
|
# Define the entry point for the docker container.
|
|
ENTRYPOINT ["C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\BuildTools\\VC\\Auxiliary\\Build\\vcvars64.bat", "&&", "cmd.exe"]
|