big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-17727. Modularize docker images (#3043)

Browse Source 
Reviewed-by: Inigo Goiri <inigoiri@apache.org>
This commit is contained in:
Gautham B A 2021-06-08 08:41:08 +05:30 committed by GitHub
parent 57a3613e5d
commit 1c0b2edde9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 1190 additions and 459 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dev-support/bin/create-release
View File

@ -514,7 +514,7 @@ function dockermode
echo "USER ${user_name}" echo "USER ${user_name}"
printf "\n\n" printf "\n\n"
) | docker build -t "${imgname}" - ) | docker build -t "${imgname}" -f - "${BASEDIR}"/dev-support/docker/
run docker run -i -t \ run docker run -i -t \
--privileged \ --privileged \

149
dev-support/docker/Dockerfile
View File

@ -32,56 +32,20 @@ RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras
ENV DEBIAN_FRONTEND noninteractive ENV DEBIAN_FRONTEND noninteractive
ENV DEBCONF_TERSE true ENV DEBCONF_TERSE true
# hadolint ignore=DL3008 ######
# Platform package dependency resolver
######
COPY pkg-resolver pkg-resolver
RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
&& chmod a+r pkg-resolver/*.json
######
# Install packages from apt
######
# hadolint ignore=DL3008,SC2046
RUN apt-get -q update \ RUN apt-get -q update \
&& apt-get -q install -y --no-install-recommends \ && apt-get -q install -y --no-install-recommends python3 \
ant \ && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:focal) \
apt-utils \
bats \
build-essential \
bzip2 \
clang \
cmake \
curl \
doxygen \
fuse \
g++ \
gcc \
git \
gnupg-agent \
hugo \
libbcprov-java \
libbz2-dev \
libcurl4-openssl-dev \
libfuse-dev \
libprotobuf-dev \
libprotoc-dev \
libsasl2-dev \
libsnappy-dev \
libssl-dev \
libtool \
libzstd-dev \
locales \
make \
maven \
nodejs \
node-yarn \
npm \
openjdk-11-jdk \
openjdk-8-jdk \
pinentry-curses \
pkg-config \
python3 \
python3-pip \
python3-pkg-resources \
python3-setuptools \
python3-wheel \
rsync \
shellcheck \
software-properties-common \
sudo \
valgrind \
zlib1g-dev \
&& apt-get clean \ && apt-get clean \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
@ -97,89 +61,16 @@ ENV MAVEN_HOME /usr
ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
####### #######
# Install SpotBugs 4.2.2 # Set env vars for SpotBugs 4.2.2
####### #######
RUN mkdir -p /opt/spotbugs \
&& curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \
-o /opt/spotbugs.tgz \
&& tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \
&& chmod +x /opt/spotbugs/bin/*
ENV SPOTBUGS_HOME /opt/spotbugs ENV SPOTBUGS_HOME /opt/spotbugs
####### #######
# Install Boost 1.72 (1.71 ships with Focal) # Set env vars for Google Protobuf 3.7.1
####### #######
# hadolint ignore=DL3003
RUN mkdir -p /opt/boost-library \
&& curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \
&& mv boost_1_72_0.tar.bz2 /opt/boost-library \
&& cd /opt/boost-library \
&& tar --bzip2 -xf boost_1_72_0.tar.bz2 \
&& cd /opt/boost-library/boost_1_72_0 \
&& ./bootstrap.sh --prefix=/usr/ \
&& ./b2 --without-python install \
&& cd /root \
&& rm -rf /opt/boost-library
######
# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal)
######
# hadolint ignore=DL3003
RUN mkdir -p /opt/protobuf-src \
&& curl -L -s -S \
https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \
-o /opt/protobuf.tar.gz \
&& tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \
&& cd /opt/protobuf-src \
&& ./configure --prefix=/opt/protobuf \
&& make "-j$(nproc)" \
&& make install \
&& cd /root \
&& rm -rf /opt/protobuf-src
ENV PROTOBUF_HOME /opt/protobuf ENV PROTOBUF_HOME /opt/protobuf
ENV PATH "${PATH}:/opt/protobuf/bin" ENV PATH "${PATH}:/opt/protobuf/bin"
####
# Install pylint and python-dateutil
####
RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1
####
# Install bower
####
# hadolint ignore=DL3008
RUN npm install -g bower@1.8.8
###
# Install hadolint
####
RUN curl -L -s -S \
https://github.com/hadolint/hadolint/releases/download/v1.11.1/hadolint-Linux-x86_64 \
-o /bin/hadolint \
&& chmod a+rx /bin/hadolint \
&& shasum -a 512 /bin/hadolint | \
awk '$1!="734e37c1f6619cbbd86b9b249e69c9af8ee1ea87a2b1ff71dccda412e9dac35e63425225a95d71572091a3f0a11e9a04c2fc25d9e91b840530c26af32b9891ca" {exit(1)}'
######
# Intel ISA-L 2.29.0
######
# hadolint ignore=DL3003,DL3008
RUN mkdir -p /opt/isa-l-src \
&& apt-get -q update \
&& apt-get install -y --no-install-recommends automake yasm \
&& apt-get clean \
&& curl -L -s -S \
https://github.com/intel/isa-l/archive/v2.29.0.tar.gz \
-o /opt/isa-l.tar.gz \
&& tar xzf /opt/isa-l.tar.gz --strip-components 1 -C /opt/isa-l-src \
&& cd /opt/isa-l-src \
&& ./autogen.sh \
&& ./configure \
&& make "-j$(nproc)" \
&& make install \
&& cd /root \
&& rm -rf /opt/isa-l-src
### ###
# Avoid out of memory errors in builds # Avoid out of memory errors in builds
### ###
@ -188,6 +79,16 @@ ENV MAVEN_OPTS -Xms256m -Xmx3072m
# Skip gpg verification when downloading Yetus via yetus-wrapper # Skip gpg verification when downloading Yetus via yetus-wrapper
ENV HADOOP_SKIP_YETUS_VERIFICATION true ENV HADOOP_SKIP_YETUS_VERIFICATION true
####
# Install packages
####
RUN pkg-resolver/install-common-pkgs.sh
RUN pkg-resolver/install-spotbugs.sh ubuntu:focal
RUN pkg-resolver/install-boost.sh ubuntu:focal
RUN pkg-resolver/install-protobuf.sh ubuntu:focal
RUN pkg-resolver/install-hadolint.sh ubuntu:focal
RUN pkg-resolver/install-intel-isa-l.sh ubuntu:focal
### ###
# Everything past this point is either not needed for testing or breaks Yetus. # Everything past this point is either not needed for testing or breaks Yetus.
# So tell Yetus not to read the rest of the file: # So tell Yetus not to read the rest of the file:

119
dev-support/docker/Dockerfile_aarch64
View File

@ -33,61 +33,19 @@ ENV DEBIAN_FRONTEND noninteractive
ENV DEBCONF_TERSE true ENV DEBCONF_TERSE true
###### ######
# Install common dependencies from packages. Versions here are either # Platform package dependency resolver
# sufficient or irrelevant.
###### ######
# hadolint ignore=DL3008 COPY pkg-resolver pkg-resolver
RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
&& chmod a+r pkg-resolver/*.json
######
# Install packages from apt
######
# hadolint ignore=DL3008,SC2046
RUN apt-get -q update \ RUN apt-get -q update \
&& apt-get -q install -y --no-install-recommends \ && apt-get -q install -y --no-install-recommends python3 \
ant \ && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:focal::arch64) \
apt-utils \
bats \
build-essential \
bzip2 \
clang \
cmake \
curl \
doxygen \
fuse \
g++ \
gcc \
git \
gnupg-agent \
hugo \
libbcprov-java \
libbz2-dev \
libcurl4-openssl-dev \
libfuse-dev \
libprotobuf-dev \
libprotoc-dev \
libsasl2-dev \
libsnappy-dev \
libssl-dev \
libtool \
libzstd-dev \
locales \
make \
maven \
nodejs \
node-yarn \
npm \
openjdk-11-jdk \
openjdk-8-jdk \
phantomjs \
pinentry-curses \
pkg-config \
python2.7 \
python3 \
python3-pip \
python3-pkg-resources \
python3-setuptools \
python3-wheel \
rsync \
shellcheck \
software-properties-common \
sudo \
valgrind \
zlib1g-dev \
&& apt-get clean \ && apt-get clean \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
@ -103,59 +61,16 @@ ENV MAVEN_HOME /usr
ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-arm64 ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-arm64
####### #######
# Install SpotBugs 4.2.2 # Set env vars for SpotBugs 4.2.2
####### #######
RUN mkdir -p /opt/spotbugs \
&& curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \
-o /opt/spotbugs.tgz \
&& tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \
&& chmod +x /opt/spotbugs/bin/*
ENV SPOTBUGS_HOME /opt/spotbugs ENV SPOTBUGS_HOME /opt/spotbugs
####### #######
# Install Boost 1.72 (1.71 ships with Focal) # Set env vars for Google Protobuf 3.7.1
####### #######
# hadolint ignore=DL3003
RUN mkdir -p /opt/boost-library \
&& curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \
&& mv boost_1_72_0.tar.bz2 /opt/boost-library \
&& cd /opt/boost-library \
&& tar --bzip2 -xf boost_1_72_0.tar.bz2 \
&& cd /opt/boost-library/boost_1_72_0 \
&& ./bootstrap.sh --prefix=/usr/ \
&& ./b2 --without-python install \
&& cd /root \
&& rm -rf /opt/boost-library
######
# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal)
######
# hadolint ignore=DL3003
RUN mkdir -p /opt/protobuf-src \
&& curl -L -s -S \
https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \
-o /opt/protobuf.tar.gz \
&& tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \
&& cd /opt/protobuf-src \
&& ./configure --prefix=/opt/protobuf \
&& make "-j$(nproc)" \
&& make install \
&& cd /root \
&& rm -rf /opt/protobuf-src
ENV PROTOBUF_HOME /opt/protobuf ENV PROTOBUF_HOME /opt/protobuf
ENV PATH "${PATH}:/opt/protobuf/bin" ENV PATH "${PATH}:/opt/protobuf/bin"
####
# Install pylint and python-dateutil
####
RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1
####
# Install bower
####
# hadolint ignore=DL3008
RUN npm install -g bower@1.8.8
### ###
# Avoid out of memory errors in builds # Avoid out of memory errors in builds
### ###
@ -167,6 +82,14 @@ ENV HADOOP_SKIP_YETUS_VERIFICATION true
# Force PhantomJS to be in 'headless' mode, do not connect to Xwindow # Force PhantomJS to be in 'headless' mode, do not connect to Xwindow
ENV QT_QPA_PLATFORM offscreen ENV QT_QPA_PLATFORM offscreen
####
# Install packages
####
RUN pkg-resolver/install-common-pkgs.sh
RUN pkg-resolver/install-spotbugs.sh ubuntu:focal::arch64
RUN pkg-resolver/install-boost.sh ubuntu:focal::arch64
RUN pkg-resolver/install-protobuf.sh ubuntu:focal::arch64
### ###
# Everything past this point is either not needed for testing or breaks Yetus. # Everything past this point is either not needed for testing or breaks Yetus.
# So tell Yetus not to read the rest of the file: # So tell Yetus not to read the rest of the file:

149
dev-support/docker/Dockerfile_centos_7
View File

@ -23,79 +23,28 @@ WORKDIR /root
SHELL ["/bin/bash", "-o", "pipefail", "-c"] SHELL ["/bin/bash", "-o", "pipefail", "-c"]
######
# Platform package dependency resolver
######
COPY pkg-resolver pkg-resolver
RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
&& chmod a+r pkg-resolver/*.json
######
# Install packages from yum
######
# hadolint ignore=DL3008,SC2046
RUN yum update -y \ RUN yum update -y \
&& yum install -y centos-release-scl \
&& yum install -y devtoolset-9 \
&& yum install -y \ && yum install -y \
ant \ centos-release-scl \
build-essential \
bzip2 \
bzip2-devel \
clang \
curl \
cyrus-sasl-devel \
doxygen \
fuse \
fuse-libs \
fuse-devel \
git \
libcurl-devel \
libtirpc-devel \
libpmem-devel \
libtool \
lz4-devel \
make \
openssl-devel \
pinentry-curses \
python3 \ python3 \
python3-pip \ && yum install -y $(pkg-resolver/resolve.py centos:7)
python3-setuptools \
python3-wheel \
rsync \
snappy-devel \
sudo \
valgrind \
zlib-devel
# Set GCC 9 as the default C/C++ compiler # Set GCC 9 as the default C/C++ compiler
RUN echo "source /opt/rh/devtoolset-9/enable" >> /etc/bashrc RUN echo "source /opt/rh/devtoolset-9/enable" >> /etc/bashrc
SHELL ["/bin/bash", "--login", "-c"] SHELL ["/bin/bash", "--login", "-c"]
#### # TODO: Set locale
# Install Maven 3.6.3
####
RUN mkdir -p /opt/maven /tmp/maven \
&& curl -L -s -S https://mirrors.estointernet.in/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz \
-o /tmp/maven/apache-maven-3.6.3-bin.tar.gz \
&& tar xzf /tmp/maven/apache-maven-3.6.3-bin.tar.gz --strip-components 1 -C /opt/maven
####
# Install CMake 3.19
####
# hadolint ignore=DL3003
RUN mkdir -p /tmp/cmake /opt/cmake \
&& curl -L -s -S https://cmake.org/files/v3.19/cmake-3.19.0.tar.gz -o /tmp/cmake/cmake-3.19.0.tar.gz \
&& tar xzf /tmp/cmake/cmake-3.19.0.tar.gz --strip-components 1 -C /opt/cmake \
&& cd /opt/cmake || exit && ./bootstrap \
&& make "-j$(nproc)" \
&& make install \
&& cd /root || exit
####
# Install zstandard
####
# hadolint ignore=DL3003
RUN mkdir -p /opt/zstd /tmp/zstd \
&& curl -L -s -S https://github.com/facebook/zstd/archive/refs/tags/v1.4.9.tar.gz -o /tmp/zstd/v1.4.9.tar.gz \
&& tar xzf /tmp/zstd/v1.4.9.tar.gz --strip-components 1 -C /opt/zstd \
&& cd /opt/zstd || exit \
&& make "-j$(nproc)" \
&& make install \
&& cd /root || exit
RUN locale-gen en_US.UTF-8
ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8'
ENV PYTHONIOENCODING=utf-8
###### ######
# Set env vars required to build Hadoop # Set env vars required to build Hadoop
@ -106,68 +55,26 @@ ENV PATH "${PATH}:${MAVEN_HOME}/bin"
ENV JAVA_HOME /usr/lib/jvm/java-1.8.0 ENV JAVA_HOME /usr/lib/jvm/java-1.8.0
####### #######
# Install SpotBugs 4.2.2 # Set env vars for SpotBugs
####### #######
RUN mkdir -p /opt/spotbugs \
&& curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \
-o /opt/spotbugs.tgz \
&& tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \
&& chmod +x /opt/spotbugs/bin/*
ENV SPOTBUGS_HOME /opt/spotbugs ENV SPOTBUGS_HOME /opt/spotbugs
####### #######
# Install Boost 1.72 (1.71 ships with Focal) # Set env vars for Google Protobuf
####### #######
# hadolint ignore=DL3003
RUN mkdir -p /opt/boost-library \
&& curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \
&& mv boost_1_72_0.tar.bz2 /opt/boost-library \
&& cd /opt/boost-library \
&& tar --bzip2 -xf boost_1_72_0.tar.bz2 \
&& cd /opt/boost-library/boost_1_72_0 \
&& ./bootstrap.sh --prefix=/usr/ \
&& ./b2 --without-python install \
&& cd /root \
&& rm -rf /opt/boost-library
######
# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal)
######
# hadolint ignore=DL3003
RUN mkdir -p /opt/protobuf-src \
&& curl -L -s -S \
https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \
-o /opt/protobuf.tar.gz \
&& tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \
&& cd /opt/protobuf-src \
&& ./configure --prefix=/opt/protobuf \
&& make "-j$(nproc)" \
&& make install \
&& cd /root \
&& rm -rf /opt/protobuf-src
ENV PROTOBUF_HOME /opt/protobuf ENV PROTOBUF_HOME /opt/protobuf
ENV PATH "${PATH}:/opt/protobuf/bin" ENV PATH "${PATH}:/opt/protobuf/bin"
####
# Install Node.js
####
# hadolint ignore=DL3003
RUN mkdir -p /tmp/node \
&& curl -L -s -S https://nodejs.org/dist/v14.16.1/node-v14.16.1.tar.gz -o /tmp/node-v14.16.1.tar.gz \
&& tar xzf /tmp/node-v14.16.1.tar.gz --strip-components 1 -C /tmp/node \
&& cd /tmp/node || exit \
&& ./configure \
&& make "-j$(nproc)" \
&& make install \
&& cd /root || exit
#### ######
# Install pylint and python-dateutil # Install packages
#### ######
RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1 RUN pkg-resolver/install-maven.sh centos:7
RUN pkg-resolver/install-cmake.sh centos:7
#### RUN pkg-resolver/install-zstandard.sh centos:7
# Install bower RUN pkg-resolver/install-yasm.sh centos:7
#### RUN pkg-resolver/install-protobuf.sh centos:7
# hadolint ignore=DL3008 RUN pkg-resolver/install-boost.sh centos:7
RUN npm install -g bower@1.8.8 RUN pkg-resolver/install-spotbugs.sh centos:7
RUN pkg-resolver/install-nodejs.sh centos:7
RUN pkg-resolver/install-common-pkgs.sh

144
dev-support/docker/Dockerfile_centos_8
View File

@ -23,48 +23,30 @@ WORKDIR /root
SHELL ["/bin/bash", "-o", "pipefail", "-c"] SHELL ["/bin/bash", "-o", "pipefail", "-c"]
######
# Platform package dependency resolver
######
COPY pkg-resolver pkg-resolver
RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
&& chmod a+r pkg-resolver/*.json
######
# Install packages from yum
######
# hadolint ignore=DL3008,SC2046
RUN yum update -y \ RUN yum update -y \
&& yum install -y \ && yum install -y python3 \
ant \ && yum install -y $(pkg-resolver/resolve.py centos:8)
bzip2 \
bzip2-devel \
clang \
curl \
cyrus-sasl-devel \
dnf \
fuse \
git \
libcurl-devel \
fuse \
fuse-libs \
fuse-devel \
libtool \
libtirpc-devel \
lz4-devel \
make \
openssl-devel \
pinentry-curses \
pkg-config \
python3 \
python3-pip \
python3-setuptools \
python3-wheel \
rsync \
sudo \
valgrind \
zlib-devel
#### ####
# Install EPEL # Install EPEL
#### ####
RUN mkdir -p /tmp/epel \ RUN pkg-resolver/install-epel.sh centos:8
&& curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm \
-o /tmp/epel/epel-release-latest-8.noarch.rpm \
&& rpm -Uvh /tmp/epel/epel-release-latest-8.noarch.rpm
RUN dnf --enablerepo=powertools install -y \ RUN dnf --enablerepo=powertools install -y \
doxygen \ doxygen \
snappy-devel snappy-devel \
yasm
RUN dnf install -y \ RUN dnf install -y \
bouncycastle \ bouncycastle \
@ -78,100 +60,34 @@ RUN dnf install -y \
RUN echo "source /opt/rh/gcc-toolset-9/enable" >> /etc/bashrc RUN echo "source /opt/rh/gcc-toolset-9/enable" >> /etc/bashrc
SHELL ["/bin/bash", "--login", "-c"] SHELL ["/bin/bash", "--login", "-c"]
#### # TODO: Set locale
# Install Maven 3.6.3
####
RUN mkdir -p /opt/maven /tmp/maven \
&& curl -L -s -S https://mirrors.estointernet.in/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz \
-o /tmp/maven/apache-maven-3.6.3-bin.tar.gz \
&& tar xzf /tmp/maven/apache-maven-3.6.3-bin.tar.gz --strip-components 1 -C /opt/maven
####
# Install CMake 3.19
####
# hadolint ignore=DL3003
RUN mkdir -p /tmp/cmake /opt/cmake \
&& curl -L -s -S https://cmake.org/files/v3.19/cmake-3.19.0.tar.gz -o /tmp/cmake/cmake-3.19.0.tar.gz \
&& tar xzf /tmp/cmake/cmake-3.19.0.tar.gz --strip-components 1 -C /opt/cmake \
&& cd /opt/cmake || exit && ./bootstrap \
&& make "-j$(nproc)" \
&& make install \
&& cd /root || exit
####
# Install zstandard
####
# hadolint ignore=DL3003
RUN mkdir -p /opt/zstd /tmp/zstd \
&& curl -L -s -S https://github.com/facebook/zstd/archive/refs/tags/v1.4.9.tar.gz -o /tmp/zstd/v1.4.9.tar.gz \
&& tar xzf /tmp/zstd/v1.4.9.tar.gz --strip-components 1 -C /opt/zstd \
&& cd /opt/zstd || exit \
&& make "-j$(nproc)" \
&& make install \
&& cd /root || exit
RUN locale-gen en_US.UTF-8
ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8'
ENV PYTHONIOENCODING=utf-8
###### ######
# Set env vars required to build Hadoop # Set env vars required to build Hadoop
###### ######
ENV MAVEN_HOME /opt/maven/apache-maven-3.6.3 ENV MAVEN_HOME /opt/maven
ENV PATH "${PATH}:${MAVEN_HOME}/bin" ENV PATH "${PATH}:${MAVEN_HOME}/bin"
# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003) # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
ENV JAVA_HOME /usr/lib/jvm/java-1.8.0 ENV JAVA_HOME /usr/lib/jvm/java-1.8.0
####### #######
# Install SpotBugs 4.2.2 # Set env vars for SpotBugs
####### #######
RUN mkdir -p /opt/spotbugs \
&& curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \
-o /opt/spotbugs.tgz \
&& tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \
&& chmod +x /opt/spotbugs/bin/*
ENV SPOTBUGS_HOME /opt/spotbugs ENV SPOTBUGS_HOME /opt/spotbugs
####### #######
# Install Boost 1.72 # Set env vars for Google Protobuf
####### #######
# hadolint ignore=DL3003
RUN mkdir -p /opt/boost-library \
&& curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \
&& mv boost_1_72_0.tar.bz2 /opt/boost-library \
&& cd /opt/boost-library \
&& tar --bzip2 -xf boost_1_72_0.tar.bz2 \
&& cd /opt/boost-library/boost_1_72_0 \
&& ./bootstrap.sh --prefix=/usr/ \
&& ./b2 --without-python install \
&& cd /root \
&& rm -rf /opt/boost-library
######
# Install Google Protobuf 3.7.1
######
# hadolint ignore=DL3003
RUN mkdir -p /opt/protobuf-src \
&& curl -L -s -S \
https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \
-o /opt/protobuf.tar.gz \
&& tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \
&& cd /opt/protobuf-src \
&& ./configure --prefix=/opt/protobuf \
&& make "-j$(nproc)" \
&& make install \
&& cd /root \
&& rm -rf /opt/protobuf-src
ENV PROTOBUF_HOME /opt/protobuf ENV PROTOBUF_HOME /opt/protobuf
ENV PATH "${PATH}:/opt/protobuf/bin" ENV PATH "${PATH}:/opt/protobuf/bin"
#### ######
# Install pylint and python-dateutil # Install packages
#### ######
RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1 RUN pkg-resolver/install-maven.sh centos:8
RUN pkg-resolver/install-cmake.sh centos:8
#### RUN pkg-resolver/install-boost.sh centos:8
# Install bower RUN pkg-resolver/install-spotbugs.sh centos:8
#### RUN pkg-resolver/install-protobuf.sh centos:8
# hadolint ignore=DL3008 RUN pkg-resolver/install-zstandard.sh centos:8
RUN npm install -g bower@1.8.8 RUN pkg-resolver/install-common-pkgs.sh

74
dev-support/docker/README.md Normal file
View File

@ -0,0 +1,74 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
# Docker images for building Hadoop
This folder contains the Dockerfiles for building Hadoop on various platforms.
# Dependency management
The mode of installation of the dependencies needed for building Hadoop varies from one platform to
the other. Different platforms have different toolchains. Some packages tend to be polymorphic
across platforms and most commonly, a package that's readily available in one platform's toolchain
isn't available on another. We thus, resort to building and installing the package from source,
causing duplication of code since this needs to be done for all the Dockerfiles pertaining to all
the platforms. We need a system to track a dependency - for a package - for a platform. Thus,
there's a lot of diversity that needs to be handled for managing package dependencies and
`pkg-resolver` caters to that.
## Supported platforms
`pkg-resolver/platforms.json` contains a list of the supported platforms for dependency management.
## Package dependencies
`pkg-resolver/packages.json` maps a dependency to a given platform. Here's the schema of this JSON.
```json
{
"dependency_1": {
"platform_1": "package_1",
"platform_2": [
"package_1",
"package_2"
]
},
"dependency_2": {
"platform_1": [
"package_1",
"package_2",
"package_3"
]
}
}
```
The root JSON element contains unique _dependency_ children. This in turn contains the name of the _
platforms_ and the list of _packages_ to be installed for that platform. Just to give an example of
how to interpret the above JSON -
1. For `dependency_1`, `package_1` needs to be installed for `platform_1`.
2. For `dependency_2`, `package_1` and `package_2` needs to be installed for `platform_2`.
3. For `dependency_2`, `package_1`, `package_3` and `package_3` needs to be installed for
`platform_1`.
## Standalone packages
Most commonly, some packages are not available across the toolchains in various platforms. Thus, we
would need to build and install them. Since we need to do this across all the Dockerfiles for all
the platforms, it could lead to code duplication and managing them becomes a hassle. Thus, we put
the build steps in a `pkg-resolver/install-<package>.sh` and invoke this in all the Dockerfiles.

50
dev-support/docker/pkg-resolver/check_platform.py Normal file
View File

@ -0,0 +1,50 @@
#!/usr/bin/env python3
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Checks whether the given platform is supported for building Apache Hadoop
"""
import json
import sys
def get_platforms():
"""
:return: A list of the supported platforms managed by pkg-resolver.
"""
with open('pkg-resolver/platforms.json', encoding='utf-8', mode='r') as platforms_file:
return json.loads(platforms_file.read())
def is_supported_platform(platform):
"""
:param platform: The name of the platform
:return: Whether the platform is supported
"""
return platform in get_platforms()
if __name__ == '__main__':
if len(sys.argv) != 2:
print('ERROR: Expecting 1 argument, {} were provided'.format(len(sys.argv) - 1),
file=sys.stderr)
sys.exit(1)
sys.exit(0 if is_supported_platform(sys.argv[1]) else 1)

56
dev-support/docker/pkg-resolver/install-boost.sh Normal file
View File

@ -0,0 +1,56 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
if [ $# -lt 1 ]; then
echo "ERROR: Need at least 1 argument, $# were provided"
exit 1
fi
pkg-resolver/check_platform.py "$1"
if [ $? -eq 1 ]; then
echo "ERROR: Unsupported platform $1"
exit 1
fi
default_version="1.72.0"
version_to_install=$default_version
if [ -n "$2" ]; then
version_to_install="$2"
fi
if [ "$version_to_install" != "1.72.0" ]; then
echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
version_to_install=$default_version
fi
if [ "$version_to_install" == "1.72.0" ]; then
# hadolint ignore=DL3003
mkdir -p /opt/boost-library &&
curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download >boost_1_72_0.tar.bz2 &&
mv boost_1_72_0.tar.bz2 /opt/boost-library &&
cd /opt/boost-library &&
tar --bzip2 -xf boost_1_72_0.tar.bz2 &&
cd /opt/boost-library/boost_1_72_0 &&
./bootstrap.sh --prefix=/usr/ &&
./b2 --without-python install &&
cd /root &&
rm -rf /opt/boost-library
else
echo "ERROR: Don't know how to install version $version_to_install"
exit 1
fi

53
dev-support/docker/pkg-resolver/install-cmake.sh Normal file
View File

@ -0,0 +1,53 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
if [ $# -lt 1 ]; then
echo "ERROR: Need at least 1 argument, $# were provided"
exit 1
fi
pkg-resolver/check_platform.py "$1"
if [ $? -eq 1 ]; then
echo "ERROR: Unsupported platform $1"
exit 1
fi
default_version="3.19.0"
version_to_install=$default_version
if [ -n "$2" ]; then
version_to_install="$2"
fi
if [ "$version_to_install" != "3.19.0" ]; then
echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
version_to_install=$default_version
fi
if [ "$version_to_install" == "3.19.0" ]; then
# hadolint ignore=DL3003
mkdir -p /tmp/cmake /opt/cmake &&
curl -L -s -S https://cmake.org/files/v3.19/cmake-3.19.0.tar.gz -o /tmp/cmake/cmake-3.19.0.tar.gz &&
tar xzf /tmp/cmake/cmake-3.19.0.tar.gz --strip-components 1 -C /opt/cmake &&
cd /opt/cmake || exit && ./bootstrap &&
make "-j$(nproc)" &&
make install &&
cd /root || exit
else
echo "ERROR: Don't know how to install version $version_to_install"
exit 1
fi

28
dev-support/docker/pkg-resolver/install-common-pkgs.sh Normal file
View File

@ -0,0 +1,28 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
######
# Install pylint and python-dateutil
######
pip3 install pylint==2.6.0 python-dateutil==2.8.1
######
# Install bower
######
# hadolint ignore=DL3008
npm install -g bower@1.8.8

49
dev-support/docker/pkg-resolver/install-epel.sh Normal file
View File

@ -0,0 +1,49 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
if [ $# -lt 1 ]; then
echo "ERROR: Need at least 1 argument, $# were provided"
exit 1
fi
pkg-resolver/check_platform.py "$1"
if [ $? -eq 1 ]; then
echo "ERROR: Unsupported platform $1"
exit 1
fi
default_version="8"
version_to_install=$default_version
if [ -n "$2" ]; then
version_to_install="$2"
fi
if [ "$version_to_install" != "8" ]; then
echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
version_to_install=$default_version
fi
if [ "$version_to_install" == "8" ]; then
mkdir -p /tmp/epel &&
curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm \
-o /tmp/epel/epel-release-latest-8.noarch.rpm &&
rpm -Uvh /tmp/epel/epel-release-latest-8.noarch.rpm
else
echo "ERROR: Don't know how to install version $version_to_install"
exit 1
fi

35
dev-support/docker/pkg-resolver/install-hadolint.sh Normal file
View File

@ -0,0 +1,35 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
if [ $# -lt 1 ]; then
echo "ERROR: Need at least 1 argument, $# were provided"
exit 1
fi
pkg-resolver/check_platform.py "$1"
if [ $? -eq 1 ]; then
echo "ERROR: Unsupported platform $1"
exit 1
fi
curl -L -s -S \
https://github.com/hadolint/hadolint/releases/download/v1.11.1/hadolint-Linux-x86_64 \
-o /bin/hadolint &&
chmod a+rx /bin/hadolint &&
shasum -a 512 /bin/hadolint |
awk '$1!="734e37c1f6619cbbd86b9b249e69c9af8ee1ea87a2b1ff71dccda412e9dac35e63425225a95d71572091a3f0a11e9a04c2fc25d9e91b840530c26af32b9891ca" {exit(1)}'

58
dev-support/docker/pkg-resolver/install-intel-isa-l.sh Normal file
View File

@ -0,0 +1,58 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
if [ $# -lt 1 ]; then
echo "ERROR: Need at least 1 argument, $# were provided"
exit 1
fi
pkg-resolver/check_platform.py "$1"
if [ $? -eq 1 ]; then
echo "ERROR: Unsupported platform $1"
exit 1
fi
default_version="2.29.0"
version_to_install=$default_version
if [ -n "$2" ]; then
version_to_install="$2"
fi
if [ "$version_to_install" != "2.29.0" ]; then
echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
version_to_install=$default_version
fi
if [ "$version_to_install" == "2.29.0" ]; then
# hadolint ignore=DL3003,DL3008
mkdir -p /opt/isa-l-src &&
curl -L -s -S \
https://github.com/intel/isa-l/archive/v2.29.0.tar.gz \
-o /opt/isa-l.tar.gz &&
tar xzf /opt/isa-l.tar.gz --strip-components 1 -C /opt/isa-l-src &&
cd /opt/isa-l-src &&
./autogen.sh &&
./configure &&
make "-j$(nproc)" &&
make install &&
cd /root &&
rm -rf /opt/isa-l-src
else
echo "ERROR: Don't know how to install version $version_to_install"
exit 1
fi

49
dev-support/docker/pkg-resolver/install-maven.sh Normal file
View File

@ -0,0 +1,49 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
if [ $# -lt 1 ]; then
echo "ERROR: Need at least 1 argument, $# were provided"
exit 1
fi
pkg-resolver/check_platform.py "$1"
if [ $? -eq 1 ]; then
echo "ERROR: Unsupported platform $1"
exit 1
fi
default_version="3.6.3"
version_to_install=$default_version
if [ -n "$2" ]; then
version_to_install="$2"
fi
if [ "$version_to_install" != "3.6.3" ]; then
echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
version_to_install=$default_version
fi
if [ "$version_to_install" == "3.6.3" ]; then
mkdir -p /opt/maven /tmp/maven &&
curl -L -s -S https://mirrors.estointernet.in/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz \
-o /tmp/maven/apache-maven-3.6.3-bin.tar.gz &&
tar xzf /tmp/maven/apache-maven-3.6.3-bin.tar.gz --strip-components 1 -C /opt/maven
else
echo "ERROR: Don't know how to install version $version_to_install"
exit 1
fi

54
dev-support/docker/pkg-resolver/install-nodejs.sh Normal file
View File

@ -0,0 +1,54 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
if [ $# -lt 1 ]; then
echo "ERROR: Need at least 1 argument, $# were provided"
exit 1
fi
pkg-resolver/check_platform.py "$1"
if [ $? -eq 1 ]; then
echo "ERROR: Unsupported platform $1"
exit 1
fi
default_version="14.16.1"
version_to_install=$default_version
if [ -n "$2" ]; then
version_to_install="$2"
fi
if [ "$version_to_install" != "14.16.1" ]; then
echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
version_to_install=$default_version
fi
if [ "$version_to_install" == "14.16.1" ]; then
# hadolint ignore=DL3003
mkdir -p /tmp/node &&
curl -L -s -S https://nodejs.org/dist/v14.16.1/node-v14.16.1.tar.gz -o /tmp/node-v14.16.1.tar.gz &&
tar xzf /tmp/node-v14.16.1.tar.gz --strip-components 1 -C /tmp/node &&
cd /tmp/node || exit &&
./configure &&
make "-j$(nproc)" &&
make install &&
cd /root || exit
else
echo "ERROR: Don't know how to install version $version_to_install"
exit 1
fi

57
dev-support/docker/pkg-resolver/install-protobuf.sh Normal file
View File

@ -0,0 +1,57 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
if [ $# -lt 1 ]; then
echo "ERROR: Need at least 1 argument, $# were provided"
exit 1
fi
pkg-resolver/check_platform.py "$1"
if [ $? -eq 1 ]; then
echo "ERROR: Unsupported platform $1"
exit 1
fi
default_version="3.7.1"
version_to_install=$default_version
if [ -n "$2" ]; then
version_to_install="$2"
fi
if [ "$version_to_install" != "3.7.1" ]; then
echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
version_to_install=$default_version
fi
if [ "$version_to_install" == "3.7.1" ]; then
# hadolint ignore=DL3003
mkdir -p /opt/protobuf-src &&
curl -L -s -S \
https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \
-o /opt/protobuf.tar.gz &&
tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src &&
cd /opt/protobuf-src &&
./configure --prefix=/opt/protobuf &&
make "-j$(nproc)" &&
make install &&
cd /root &&
rm -rf /opt/protobuf-src
else
echo "ERROR: Don't know how to install version $version_to_install"
exit 1
fi

50
dev-support/docker/pkg-resolver/install-spotbugs.sh Normal file
View File

@ -0,0 +1,50 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
if [ $# -lt 1 ]; then
echo "ERROR: Need at least 1 argument, $# were provided"
exit 1
fi
pkg-resolver/check_platform.py "$1"
if [ $? -eq 1 ]; then
echo "ERROR: Unsupported platform $1"
exit 1
fi
default_version="4.2.2"
version_to_install=$default_version
if [ -n "$2" ]; then
version_to_install="$2"
fi
if [ "$version_to_install" != "4.2.2" ]; then
echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
version_to_install=$default_version
fi
if [ "$version_to_install" == "4.2.2" ]; then
mkdir -p /opt/spotbugs &&
curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \
-o /opt/spotbugs.tgz &&
tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs &&
chmod +x /opt/spotbugs/bin/*
else
echo "ERROR: Don't know how to install version $version_to_install"
exit 1
fi

49
dev-support/docker/pkg-resolver/install-yasm.sh Normal file
View File

@ -0,0 +1,49 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
if [ $# -lt 1 ]; then
echo "ERROR: Need at least 1 argument, $# were provided"
exit 1
fi
pkg-resolver/check_platform.py "$1"
if [ $? -eq 1 ]; then
echo "ERROR: Unsupported platform $1"
exit 1
fi
default_version="1.2.0-4"
version_to_install=$default_version
if [ -n "$2" ]; then
version_to_install="$2"
fi
if [ "$version_to_install" != "1.2.0-4" ]; then
echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
version_to_install=$default_version
fi
if [ "$version_to_install" == "1.2.0-4" ]; then
mkdir -p /tmp/yasm &&
curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/y/yasm-1.2.0-4.el7.x86_64.rpm \
-o /tmp/yasm-1.2.0-4.el7.x86_64.rpm &&
rpm -Uvh /tmp/yasm-1.2.0-4.el7.x86_64.rpm
else
echo "ERROR: Don't know how to install version $version_to_install"
exit 1
fi

53
dev-support/docker/pkg-resolver/install-zstandard.sh Normal file
View File

@ -0,0 +1,53 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
if [ $# -lt 1 ]; then
echo "ERROR: Need at least 1 argument, $# were provided"
exit 1
fi
pkg-resolver/check_platform.py "$1"
if [ $? -eq 1 ]; then
echo "ERROR: Unsupported platform $1"
exit 1
fi
default_version="1.4.9"
version_to_install=$default_version
if [ -n "$2" ]; then
version_to_install="$2"
fi
if [ "$version_to_install" != "1.4.9" ]; then
echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
version_to_install=$default_version
fi
if [ "$version_to_install" == "1.4.9" ]; then
# hadolint ignore=DL3003
mkdir -p /opt/zstd /tmp/zstd &&
curl -L -s -S https://github.com/facebook/zstd/archive/refs/tags/v1.4.9.tar.gz -o /tmp/zstd/v1.4.9.tar.gz &&
tar xzf /tmp/zstd/v1.4.9.tar.gz --strip-components 1 -C /opt/zstd &&
cd /opt/zstd || exit &&
make "-j$(nproc)" &&
make install &&
cd /root || exit
else
echo "ERROR: Don't know how to install version $version_to_install"
exit 1
fi

302
dev-support/docker/pkg-resolver/packages.json Normal file
View File

@ -0,0 +1,302 @@
{
"ant": {
"ubuntu:focal": "ant",
"ubuntu:focal::arch64": "ant",
"centos:7": "ant",
"centos:8": "ant"
},
"apt-utils": {
"ubuntu:focal": "apt-utils",
"ubuntu:focal::arch64": "apt-utils"
},
"automake": {
"ubuntu:focal": "automake",
"ubuntu:focal::arch64": "automake",
"centos:7": "automake",
"centos:8": "automake"
},
"bats": {
"ubuntu:focal": "bats",
"ubuntu:focal::arch64": "bats"
},
"build-essential": {
"ubuntu:focal": "build-essential",
"ubuntu:focal::arch64": "build-essential",
"centos:7": "build-essential"
},
"bzip2": {
"ubuntu:focal": [
"bzip2",
"libbz2-dev"
],
"ubuntu:focal::arch64": [
"bzip2",
"libbz2-dev"
],
"centos:7": [
"bzip2",
"bzip2-devel"
],
"centos:8": [
"bzip2",
"bzip2-devel"
]
},
"clang": {
"ubuntu:focal": "clang",
"ubuntu:focal::arch64": "clang",
"centos:7": "clang",
"centos:8": "clang"
},
"cmake": {
"ubuntu:focal": "cmake",
"ubuntu:focal::arch64": "cmake"
},
"curl": {
"ubuntu:focal": [
"curl",
"libcurl4-openssl-dev"
],
"ubuntu:focal::arch64": [
"curl",
"libcurl4-openssl-dev"
],
"centos:7": [
"curl",
"libcurl-devel"
],
"centos:8": [
"curl",
"libcurl-devel"
]
},
"doxygen": {
"ubuntu:focal": "doxygen",
"ubuntu:focal::arch64": "doxygen",
"centos:7": "doxygen"
},
"dnf": {
"centos:8": "dnf"
},
"fuse": {
"ubuntu:focal": [
"fuse",
"libfuse-dev"
],
"ubuntu:focal::arch64": [
"fuse",
"libfuse-dev"
],
"centos:7": [
"fuse",
"fuse-libs",
"fuse-devel"
],
"centos:8": [
"fuse",
"fuse-libs",
"fuse-devel"
]
},
"gcc": {
"ubuntu:focal": [
"gcc",
"g++"
],
"ubuntu:focal::arch64": [
"gcc",
"g++"
],
"centos:7": [
"centos-release-scl",
"devtoolset-9"
]
},
"git": {
"ubuntu:focal": "git",
"ubuntu:focal::arch64": "git",
"centos:7": "git",
"centos:8": "git"
},
"gnupg-agent": {
"ubuntu:focal": "gnupg-agent",
"ubuntu:focal::arch64": "gnupg-agent"
},
"hugo": {
"ubuntu:focal": "hugo",
"ubuntu:focal::arch64": "hugo"
},
"libbcprov-java": {
"ubuntu:focal": "libbcprov-java",
"ubuntu:focal::arch64": "libbcprov-java"
},
"libtool": {
"ubuntu:focal": "libtool",
"ubuntu:focal::arch64": "libtool",
"centos:7": "libtool",
"centos:8": "libtool"
},
"openssl": {
"ubuntu:focal": "libssl-dev",
"ubuntu:focal::arch64": "libssl-dev",
"centos:7": "openssl-devel",
"centos:8": "openssl-devel"
},
"protocol-buffers": {
"ubuntu:focal": [
"libprotobuf-dev",
"libprotoc-dev"
],
"ubuntu:focal::arch64": [
"libprotobuf-dev",
"libprotoc-dev"
]
},
"sasl": {
"ubuntu:focal": "libsasl2-dev",
"ubuntu:focal::arch64": "libsasl2-dev",
"centos:7": "cyrus-sasl-devel",
"centos:8": "cyrus-sasl-devel"
},
"snappy": {
"ubuntu:focal": "libsnappy-dev",
"ubuntu:focal::arch64": "libsnappy-dev",
"centos:7": "snappy-devel"
},
"zlib": {
"ubuntu:focal": [
"libzstd-dev",
"zlib1g-dev"
],
"ubuntu:focal::arch64": [
"libzstd-dev",
"zlib1g-dev"
],
"centos:7": [
"zlib-devel",
"lz4-devel"
],
"centos:8": [
"zlib-devel",
"lz4-devel"
]
},
"locales": {
"ubuntu:focal": "locales",
"ubuntu:focal::arch64": "locales"
},
"libtirpc-devel": {
"centos:7": "libtirpc-devel",
"centos:8": "libtirpc-devel"
},
"libpmem": {
"centos:7": "libpmem-devel"
},
"make": {
"ubuntu:focal": "make",
"ubuntu:focal::arch64": "make",
"centos:7": "make",
"centos:8": "make"
},
"maven": {
"ubuntu:focal": "maven",
"ubuntu:focal::arch64": "maven"
},
"javascript": {
"ubuntu:focal": [
"nodejs",
"node-yarn",
"npm"
],
"ubuntu:focal::arch64": [
"nodejs",
"node-yarn",
"npm",
"phantomjs"
]
},
"java": {
"ubuntu:focal": [
"openjdk-8-jdk",
"openjdk-11-jdk"
],
"ubuntu:focal::arch64": [
"openjdk-8-jdk",
"openjdk-11-jdk"
]
},
"pinentry-curses": {
"ubuntu:focal": "pinentry-curses",
"ubuntu:focal::arch64": "pinentry-curses",
"centos:7": "pinentry-curses",
"centos:8": "pinentry-curses"
},
"pkg-config": {
"ubuntu:focal": "pkg-config",
"ubuntu:focal::arch64": "pkg-config",
"centos:8": "pkg-config"
},
"python": {
"ubuntu:focal": [
"python3",
"python3-pip",
"python3-pkg-resources",
"python3-setuptools",
"python3-wheel"
],
"ubuntu:focal::arch64": [
"python2.7",
"python3",
"python3-pip",
"python3-pkg-resources",
"python3-setuptools",
"python3-wheel"
],
"centos:7": [
"python3",
"python3-pip",
"python3-setuptools",
"python3-wheel"
],
"centos:8": [
"python3",
"python3-pip",
"python3-setuptools",
"python3-wheel"
]
},
"rsync": {
"ubuntu:focal": "rsync",
"ubuntu:focal::arch64": "rsync",
"centos:7": "rsync",
"centos:8": "rsync"
},
"shellcheck": {
"ubuntu:focal": "shellcheck",
"ubuntu:focal::arch64": "shellcheck"
},
"shasum": {
"centos:7": "perl-Digest-SHA",
"centos:8": "perl-Digest-SHA"
},
"software-properties-common": {
"ubuntu:focal": "software-properties-common",
"ubuntu:focal::arch64": "software-properties-common"
},
"sudo": {
"ubuntu:focal": "sudo",
"ubuntu:focal::arch64": "sudo",
"centos:7": "sudo",
"centos:8": "sudo"
},
"valgrind": {
"ubuntu:focal": "valgrind",
"ubuntu:focal::arch64": "valgrind",
"centos:7": "valgrind",
"centos:8": "valgrind"
},
"yasm": {
"ubuntu:focal": "yasm",
"ubuntu:focal::arch64": "yasm"
}
}

6
dev-support/docker/pkg-resolver/platforms.json Normal file
View File

@ -0,0 +1,6 @@
[
"ubuntu:focal",
"ubuntu:focal::arch64",
"centos:7",
"centos:8"
]

61
dev-support/docker/pkg-resolver/resolve.py Normal file
View File

@ -0,0 +1,61 @@
#!/usr/bin/env python3
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Platform package dependency resolver for building Apache Hadoop.
"""
import json
import sys
from check_platform import is_supported_platform
def get_packages(platform):
"""
Resolve and get the list of packages to install for the given platform.
:param platform: The platform for which the packages needs to be resolved.
:return: A list of resolved packages to install.
"""
with open('pkg-resolver/packages.json', encoding='utf-8', mode='r') as pkg_file:
pkgs = json.loads(pkg_file.read())
packages = []
for platforms in filter(lambda x: x.get(platform) is not None, pkgs.values()):
if isinstance(platforms.get(platform), list):
packages.extend(platforms.get(platform))
else:
packages.append(platforms.get(platform))
return packages
if __name__ == '__main__':
if len(sys.argv) < 2:
print('ERROR: Need at least 1 argument, {} were provided'.format(len(sys.argv) - 1),
file=sys.stderr)
sys.exit(1)
platform_arg = sys.argv[1]
if not is_supported_platform(platform_arg):
print(
'ERROR: The given platform {} is not supported. '
'Please refer to platforms.json for a list of supported platforms'.format(
platform_arg), file=sys.stderr)
sys.exit(1)
packages_to_install = get_packages(platform_arg)
print(' '.join(packages_to_install))