HADOOP-16797. Add Dockerfile for ARM builds. Contributed by Vinayakumar B. (#1801)

This commit is contained in:
Vinayakumar B 2020-01-13 10:40:29 +05:30 committed by GitHub
parent d842dfffa5
commit 52b360a928
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 265 additions and 3 deletions

View File

@ -204,6 +204,11 @@ function set_defaults
DOCKERFILE="${BASEDIR}/dev-support/docker/Dockerfile"
DOCKERRAN=false
CPU_ARCH=$(echo "$MACHTYPE" | cut -d- -f1)
if [ "$CPU_ARCH" = "aarch64" ]; then
DOCKERFILE="${BASEDIR}/dev-support/docker/Dockerfile_aarch64"
fi
# Extract Java version from ${BASEDIR}/pom.xml
# doing this outside of maven means we can do this before
# the docker container comes up...
@ -249,7 +254,9 @@ function startgpgagent
eval $("${GPGAGENT}" --daemon \
--options "${LOGDIR}/gpgagent.conf" \
--log-file="${LOGDIR}/create-release-gpgagent.log")
GPGAGENTPID=$(echo "${GPG_AGENT_INFO}" | cut -f 2 -d:)
GPGAGENTPID=$(pgrep "${GPGAGENT}")
GPG_AGENT_INFO="$HOME/.gnupg/S.gpg-agent:$GPGAGENTPID:1"
export GPG_AGENT_INFO
fi
if [[ -n "${GPG_AGENT_INFO}" ]]; then
@ -499,7 +506,12 @@ function dockermode
# we always force build with the OpenJDK JDK
# but with the correct version
echo "ENV JAVA_HOME /usr/lib/jvm/java-${JVM_VERSION}-openjdk-amd64"
if [ "$CPU_ARCH" = "aarch64" ]; then
echo "ENV JAVA_HOME /usr/lib/jvm/java-${JVM_VERSION}-openjdk-arm64"
else
echo "ENV JAVA_HOME /usr/lib/jvm/java-${JVM_VERSION}-openjdk-amd64"
fi
echo "USER ${user_name}"
printf "\n\n"
) | docker build -t "${imgname}" -

View File

@ -0,0 +1,235 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Dockerfile for installing the necessary dependencies for building Hadoop.
# See BUILDING.txt.
FROM ubuntu:xenial
WORKDIR /root
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
#####
# Disable suggests/recommends
#####
RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras
RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras
ENV DEBIAN_FRONTEND noninteractive
ENV DEBCONF_TERSE true
######
# Install common dependencies from packages. Versions here are either
# sufficient or irrelevant.
#
# WARNING: DO NOT PUT JAVA APPS HERE! Otherwise they will install default
# Ubuntu Java. See Java section below!
######
# hadolint ignore=DL3008
RUN apt-get -q update \
&& apt-get -q install -y --no-install-recommends \
apt-utils \
build-essential \
bzip2 \
clang \
curl \
doxygen \
fuse \
g++ \
gcc \
git \
gnupg-agent \
libbz2-dev \
libcurl4-openssl-dev \
libfuse-dev \
libprotobuf-dev \
libprotoc-dev \
libsasl2-dev \
libsnappy-dev \
libssl-dev \
libtool \
libzstd1-dev \
locales \
make \
pinentry-curses \
pkg-config \
python \
python2.7 \
python-pip \
python-pkg-resources \
python-setuptools \
python-wheel \
rsync \
software-properties-common \
snappy \
sudo \
valgrind \
zlib1g-dev \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
#######
# OpenJDK 8
#######
# hadolint ignore=DL3008
RUN apt-get -q update \
&& apt-get -q install -y --no-install-recommends openjdk-8-jdk libbcprov-java \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
######
# Install cmake 3.1.0 (3.5.1 ships with Xenial)
# There is no cmake binary available for aarch64. Build from source.
######
# hadolint ignore=DL3003
RUN mkdir -p /opt/cmake/src \
&& curl -L -s -S \
https://cmake.org/files/v3.1/cmake-3.1.0-1-src.tar.bz2 \
-o /opt/cmake/cmake-src.tar.bz2 \
&& tar xvjf /opt/cmake/cmake-src.tar.bz2 -C /opt/cmake/src \
&& cd /opt/cmake/src \
&& tar xvjf cmake-3.1.0.tar.bz2 \
&& cd cmake-3.1.0 && patch -p0 -i ../cmake-3.1.0-1.patch && mkdir .build && cd .build \
&& ../bootstrap --parallel=2 \
&& make -j2 && ./bin/cpack \
&& tar xzf cmake-3.1.0-Linux-aarch64.tar.gz --strip-components 1 -C /opt/cmake \
&& cd /opt/cmake && rm -rf /opt/cmake/src
ENV CMAKE_HOME /opt/cmake
ENV PATH "${PATH}:/opt/cmake/bin"
######
# Install Google Protobuf 3.7.1 (2.6.0 ships with Xenial)
######
# hadolint ignore=DL3003
RUN mkdir -p /opt/protobuf-src \
&& curl -L -s -S \
https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \
-o /opt/protobuf.tar.gz \
&& tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \
&& cd /opt/protobuf-src \
&& ./configure --prefix=/opt/protobuf \
&& make install \
&& cd /root \
&& rm -rf /opt/protobuf-src
ENV PROTOBUF_HOME /opt/protobuf
ENV PATH "${PATH}:/opt/protobuf/bin"
######
# Install Apache Maven 3.3.9 (3.3.9 ships with Xenial)
######
# hadolint ignore=DL3008
RUN apt-get -q update \
&& apt-get -q install -y --no-install-recommends maven \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
ENV MAVEN_HOME /usr
######
# Install findbugs 3.0.1 (3.0.1 ships with Xenial)
# Ant is needed for findbugs
######
# hadolint ignore=DL3008
RUN apt-get -q update \
&& apt-get -q install -y --no-install-recommends findbugs ant \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
ENV FINDBUGS_HOME /usr
####
# Install shellcheck (0.4.6, the latest as of 2017-09-26)
####
# hadolint ignore=DL3008
RUN add-apt-repository -y ppa:hvr/ghc \
&& apt-get -q update \
&& apt-get -q install -y --no-install-recommends shellcheck \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
####
# Install bats (0.4.0, the latest as of 2017-09-26, ships with Xenial)
####
# hadolint ignore=DL3008
RUN apt-get -q update \
&& apt-get -q install -y --no-install-recommends bats \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
####
# Install pylint at fixed version (2.0.0 removed python2 support)
# https://github.com/PyCQA/pylint/issues/2294
####
RUN pip2 install pylint==1.9.2
####
# Install dateutil.parser
####
RUN pip2 install python-dateutil==2.7.3
###
# Install node.js 8.17.0 for web UI framework (4.2.6 ships with Xenial)
###
RUN curl -L -s -S https://deb.nodesource.com/setup_8.x | bash - \
&& apt-get install -y --no-install-recommends nodejs=8.17.0-1nodesource1 \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* \
&& npm install -g bower@1.8.8
###
## Install Yarn 1.12.1 for web UI framework
####
RUN curl -s -S https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
&& echo 'deb https://dl.yarnpkg.com/debian/ stable main' > /etc/apt/sources.list.d/yarn.list \
&& apt-get -q update \
&& apt-get install -y --no-install-recommends yarn=1.21.1-1 \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
###
# Install phantomjs built for aarch64
####
RUN mkdir -p /opt/phantomjs \
&& curl -L -s -S \
https://github.com/liusheng/phantomjs/releases/download/2.1.1/phantomjs-2.1.1-linux-aarch64.tar.bz2 \
-o /opt/phantomjs/phantomjs-2.1.1-linux-aarch64.tar.bz2 \
&& tar xvjf /opt/phantomjs/phantomjs-2.1.1-linux-aarch64.tar.bz2 --strip-components 1 -C /opt/phantomjs \
&& cp /opt/phantomjs/bin/phantomjs /usr/bin/ \
&& rm -rf /opt/phantomjs
###
# Avoid out of memory errors in builds
###
ENV MAVEN_OPTS -Xms256m -Xmx1536m
###
# Everything past this point is either not needed for testing or breaks Yetus.
# So tell Yetus not to read the rest of the file:
# YETUS CUT HERE
###
# Hugo static website generator (for new hadoop site docs)
RUN curl -L -o hugo.deb https://github.com/gohugoio/hugo/releases/download/v0.58.3/hugo_0.58.3_Linux-ARM64.deb \
&& dpkg --install hugo.deb \
&& rm hugo.deb
# Add a welcome message and environment checks.
COPY hadoop_env_checks.sh /root/hadoop_env_checks.sh
RUN chmod 755 /root/hadoop_env_checks.sh
# hadolint ignore=SC2016
RUN echo '${HOME}/hadoop_env_checks.sh' >> /root/.bashrc

View File

@ -142,6 +142,12 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<artifactId>findbugs</artifactId>
<version>3.0.1</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>xml-apis</groupId>
<artifactId>xml-apis</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>

View File

@ -19,7 +19,15 @@ set -e # exit on error
cd "$(dirname "$0")" # connect to root
docker build -t hadoop-build dev-support/docker
DOCKER_DIR=dev-support/docker
DOCKER_FILE="${DOCKER_DIR}/Dockerfile"
CPU_ARCH=$(echo "$MACHTYPE" | cut -d- -f1)
if [ "$CPU_ARCH" = "aarch64" ]; then
DOCKER_FILE="${DOCKER_DIR}/Dockerfile_aarch64"
fi
docker build -t hadoop-build -f $DOCKER_FILE $DOCKER_DIR
USER_NAME=${SUDO_USER:=$USER}
USER_ID=$(id -u "${USER_NAME}")
@ -80,5 +88,6 @@ docker run --rm=true $DOCKER_INTERACTIVE_RUN \
-v "${PWD}:/home/${USER_NAME}/hadoop${V_OPTS:-}" \
-w "/home/${USER_NAME}/hadoop" \
-v "${HOME}/.m2:/home/${USER_NAME}/.m2${V_OPTS:-}" \
-v "${HOME}/.gnupg:/home/${USER_NAME}/.gnupg${V_OPTS:-}" \
-u "${USER_NAME}" \
"hadoop-build-${USER_ID}" "$@"