HADOOP-16797. Add Dockerfile for ARM builds. Contributed by Vinayakumar B. (#1801)
This commit is contained in:
parent
d842dfffa5
commit
52b360a928
@ -204,6 +204,11 @@ function set_defaults
|
||||
DOCKERFILE="${BASEDIR}/dev-support/docker/Dockerfile"
|
||||
DOCKERRAN=false
|
||||
|
||||
CPU_ARCH=$(echo "$MACHTYPE" | cut -d- -f1)
|
||||
if [ "$CPU_ARCH" = "aarch64" ]; then
|
||||
DOCKERFILE="${BASEDIR}/dev-support/docker/Dockerfile_aarch64"
|
||||
fi
|
||||
|
||||
# Extract Java version from ${BASEDIR}/pom.xml
|
||||
# doing this outside of maven means we can do this before
|
||||
# the docker container comes up...
|
||||
@ -249,7 +254,9 @@ function startgpgagent
|
||||
eval $("${GPGAGENT}" --daemon \
|
||||
--options "${LOGDIR}/gpgagent.conf" \
|
||||
--log-file="${LOGDIR}/create-release-gpgagent.log")
|
||||
GPGAGENTPID=$(echo "${GPG_AGENT_INFO}" | cut -f 2 -d:)
|
||||
GPGAGENTPID=$(pgrep "${GPGAGENT}")
|
||||
GPG_AGENT_INFO="$HOME/.gnupg/S.gpg-agent:$GPGAGENTPID:1"
|
||||
export GPG_AGENT_INFO
|
||||
fi
|
||||
|
||||
if [[ -n "${GPG_AGENT_INFO}" ]]; then
|
||||
@ -499,7 +506,12 @@ function dockermode
|
||||
|
||||
# we always force build with the OpenJDK JDK
|
||||
# but with the correct version
|
||||
echo "ENV JAVA_HOME /usr/lib/jvm/java-${JVM_VERSION}-openjdk-amd64"
|
||||
if [ "$CPU_ARCH" = "aarch64" ]; then
|
||||
echo "ENV JAVA_HOME /usr/lib/jvm/java-${JVM_VERSION}-openjdk-arm64"
|
||||
else
|
||||
echo "ENV JAVA_HOME /usr/lib/jvm/java-${JVM_VERSION}-openjdk-amd64"
|
||||
fi
|
||||
|
||||
echo "USER ${user_name}"
|
||||
printf "\n\n"
|
||||
) | docker build -t "${imgname}" -
|
||||
|
235
dev-support/docker/Dockerfile_aarch64
Normal file
235
dev-support/docker/Dockerfile_aarch64
Normal file
@ -0,0 +1,235 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Dockerfile for installing the necessary dependencies for building Hadoop.
|
||||
# See BUILDING.txt.
|
||||
|
||||
FROM ubuntu:xenial
|
||||
|
||||
WORKDIR /root
|
||||
|
||||
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
|
||||
|
||||
#####
|
||||
# Disable suggests/recommends
|
||||
#####
|
||||
RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras
|
||||
RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras
|
||||
|
||||
ENV DEBIAN_FRONTEND noninteractive
|
||||
ENV DEBCONF_TERSE true
|
||||
|
||||
######
|
||||
# Install common dependencies from packages. Versions here are either
|
||||
# sufficient or irrelevant.
|
||||
#
|
||||
# WARNING: DO NOT PUT JAVA APPS HERE! Otherwise they will install default
|
||||
# Ubuntu Java. See Java section below!
|
||||
######
|
||||
# hadolint ignore=DL3008
|
||||
RUN apt-get -q update \
|
||||
&& apt-get -q install -y --no-install-recommends \
|
||||
apt-utils \
|
||||
build-essential \
|
||||
bzip2 \
|
||||
clang \
|
||||
curl \
|
||||
doxygen \
|
||||
fuse \
|
||||
g++ \
|
||||
gcc \
|
||||
git \
|
||||
gnupg-agent \
|
||||
libbz2-dev \
|
||||
libcurl4-openssl-dev \
|
||||
libfuse-dev \
|
||||
libprotobuf-dev \
|
||||
libprotoc-dev \
|
||||
libsasl2-dev \
|
||||
libsnappy-dev \
|
||||
libssl-dev \
|
||||
libtool \
|
||||
libzstd1-dev \
|
||||
locales \
|
||||
make \
|
||||
pinentry-curses \
|
||||
pkg-config \
|
||||
python \
|
||||
python2.7 \
|
||||
python-pip \
|
||||
python-pkg-resources \
|
||||
python-setuptools \
|
||||
python-wheel \
|
||||
rsync \
|
||||
software-properties-common \
|
||||
snappy \
|
||||
sudo \
|
||||
valgrind \
|
||||
zlib1g-dev \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
|
||||
#######
|
||||
# OpenJDK 8
|
||||
#######
|
||||
# hadolint ignore=DL3008
|
||||
RUN apt-get -q update \
|
||||
&& apt-get -q install -y --no-install-recommends openjdk-8-jdk libbcprov-java \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
|
||||
######
|
||||
# Install cmake 3.1.0 (3.5.1 ships with Xenial)
|
||||
# There is no cmake binary available for aarch64. Build from source.
|
||||
######
|
||||
# hadolint ignore=DL3003
|
||||
RUN mkdir -p /opt/cmake/src \
|
||||
&& curl -L -s -S \
|
||||
https://cmake.org/files/v3.1/cmake-3.1.0-1-src.tar.bz2 \
|
||||
-o /opt/cmake/cmake-src.tar.bz2 \
|
||||
&& tar xvjf /opt/cmake/cmake-src.tar.bz2 -C /opt/cmake/src \
|
||||
&& cd /opt/cmake/src \
|
||||
&& tar xvjf cmake-3.1.0.tar.bz2 \
|
||||
&& cd cmake-3.1.0 && patch -p0 -i ../cmake-3.1.0-1.patch && mkdir .build && cd .build \
|
||||
&& ../bootstrap --parallel=2 \
|
||||
&& make -j2 && ./bin/cpack \
|
||||
&& tar xzf cmake-3.1.0-Linux-aarch64.tar.gz --strip-components 1 -C /opt/cmake \
|
||||
&& cd /opt/cmake && rm -rf /opt/cmake/src
|
||||
ENV CMAKE_HOME /opt/cmake
|
||||
ENV PATH "${PATH}:/opt/cmake/bin"
|
||||
|
||||
######
|
||||
# Install Google Protobuf 3.7.1 (2.6.0 ships with Xenial)
|
||||
######
|
||||
# hadolint ignore=DL3003
|
||||
RUN mkdir -p /opt/protobuf-src \
|
||||
&& curl -L -s -S \
|
||||
https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \
|
||||
-o /opt/protobuf.tar.gz \
|
||||
&& tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \
|
||||
&& cd /opt/protobuf-src \
|
||||
&& ./configure --prefix=/opt/protobuf \
|
||||
&& make install \
|
||||
&& cd /root \
|
||||
&& rm -rf /opt/protobuf-src
|
||||
ENV PROTOBUF_HOME /opt/protobuf
|
||||
ENV PATH "${PATH}:/opt/protobuf/bin"
|
||||
|
||||
######
|
||||
# Install Apache Maven 3.3.9 (3.3.9 ships with Xenial)
|
||||
######
|
||||
# hadolint ignore=DL3008
|
||||
RUN apt-get -q update \
|
||||
&& apt-get -q install -y --no-install-recommends maven \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
ENV MAVEN_HOME /usr
|
||||
|
||||
######
|
||||
# Install findbugs 3.0.1 (3.0.1 ships with Xenial)
|
||||
# Ant is needed for findbugs
|
||||
######
|
||||
# hadolint ignore=DL3008
|
||||
RUN apt-get -q update \
|
||||
&& apt-get -q install -y --no-install-recommends findbugs ant \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
ENV FINDBUGS_HOME /usr
|
||||
|
||||
####
|
||||
# Install shellcheck (0.4.6, the latest as of 2017-09-26)
|
||||
####
|
||||
# hadolint ignore=DL3008
|
||||
RUN add-apt-repository -y ppa:hvr/ghc \
|
||||
&& apt-get -q update \
|
||||
&& apt-get -q install -y --no-install-recommends shellcheck \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
####
|
||||
# Install bats (0.4.0, the latest as of 2017-09-26, ships with Xenial)
|
||||
####
|
||||
# hadolint ignore=DL3008
|
||||
RUN apt-get -q update \
|
||||
&& apt-get -q install -y --no-install-recommends bats \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
####
|
||||
# Install pylint at fixed version (2.0.0 removed python2 support)
|
||||
# https://github.com/PyCQA/pylint/issues/2294
|
||||
####
|
||||
RUN pip2 install pylint==1.9.2
|
||||
|
||||
####
|
||||
# Install dateutil.parser
|
||||
####
|
||||
RUN pip2 install python-dateutil==2.7.3
|
||||
|
||||
###
|
||||
# Install node.js 8.17.0 for web UI framework (4.2.6 ships with Xenial)
|
||||
###
|
||||
RUN curl -L -s -S https://deb.nodesource.com/setup_8.x | bash - \
|
||||
&& apt-get install -y --no-install-recommends nodejs=8.17.0-1nodesource1 \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& npm install -g bower@1.8.8
|
||||
|
||||
###
|
||||
## Install Yarn 1.12.1 for web UI framework
|
||||
####
|
||||
RUN curl -s -S https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
|
||||
&& echo 'deb https://dl.yarnpkg.com/debian/ stable main' > /etc/apt/sources.list.d/yarn.list \
|
||||
&& apt-get -q update \
|
||||
&& apt-get install -y --no-install-recommends yarn=1.21.1-1 \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
###
|
||||
# Install phantomjs built for aarch64
|
||||
####
|
||||
RUN mkdir -p /opt/phantomjs \
|
||||
&& curl -L -s -S \
|
||||
https://github.com/liusheng/phantomjs/releases/download/2.1.1/phantomjs-2.1.1-linux-aarch64.tar.bz2 \
|
||||
-o /opt/phantomjs/phantomjs-2.1.1-linux-aarch64.tar.bz2 \
|
||||
&& tar xvjf /opt/phantomjs/phantomjs-2.1.1-linux-aarch64.tar.bz2 --strip-components 1 -C /opt/phantomjs \
|
||||
&& cp /opt/phantomjs/bin/phantomjs /usr/bin/ \
|
||||
&& rm -rf /opt/phantomjs
|
||||
|
||||
###
|
||||
# Avoid out of memory errors in builds
|
||||
###
|
||||
ENV MAVEN_OPTS -Xms256m -Xmx1536m
|
||||
|
||||
###
|
||||
# Everything past this point is either not needed for testing or breaks Yetus.
|
||||
# So tell Yetus not to read the rest of the file:
|
||||
# YETUS CUT HERE
|
||||
###
|
||||
|
||||
# Hugo static website generator (for new hadoop site docs)
|
||||
RUN curl -L -o hugo.deb https://github.com/gohugoio/hugo/releases/download/v0.58.3/hugo_0.58.3_Linux-ARM64.deb \
|
||||
&& dpkg --install hugo.deb \
|
||||
&& rm hugo.deb
|
||||
|
||||
|
||||
# Add a welcome message and environment checks.
|
||||
COPY hadoop_env_checks.sh /root/hadoop_env_checks.sh
|
||||
RUN chmod 755 /root/hadoop_env_checks.sh
|
||||
# hadolint ignore=SC2016
|
||||
RUN echo '${HOME}/hadoop_env_checks.sh' >> /root/.bashrc
|
@ -142,6 +142,12 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<artifactId>findbugs</artifactId>
|
||||
<version>3.0.1</version>
|
||||
<scope>provided</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>xml-apis</groupId>
|
||||
<artifactId>xml-apis</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
|
@ -19,7 +19,15 @@ set -e # exit on error
|
||||
|
||||
cd "$(dirname "$0")" # connect to root
|
||||
|
||||
docker build -t hadoop-build dev-support/docker
|
||||
DOCKER_DIR=dev-support/docker
|
||||
DOCKER_FILE="${DOCKER_DIR}/Dockerfile"
|
||||
|
||||
CPU_ARCH=$(echo "$MACHTYPE" | cut -d- -f1)
|
||||
if [ "$CPU_ARCH" = "aarch64" ]; then
|
||||
DOCKER_FILE="${DOCKER_DIR}/Dockerfile_aarch64"
|
||||
fi
|
||||
|
||||
docker build -t hadoop-build -f $DOCKER_FILE $DOCKER_DIR
|
||||
|
||||
USER_NAME=${SUDO_USER:=$USER}
|
||||
USER_ID=$(id -u "${USER_NAME}")
|
||||
@ -80,5 +88,6 @@ docker run --rm=true $DOCKER_INTERACTIVE_RUN \
|
||||
-v "${PWD}:/home/${USER_NAME}/hadoop${V_OPTS:-}" \
|
||||
-w "/home/${USER_NAME}/hadoop" \
|
||||
-v "${HOME}/.m2:/home/${USER_NAME}/.m2${V_OPTS:-}" \
|
||||
-v "${HOME}/.gnupg:/home/${USER_NAME}/.gnupg${V_OPTS:-}" \
|
||||
-u "${USER_NAME}" \
|
||||
"hadoop-build-${USER_ID}" "$@"
|
||||
|
Loading…
Reference in New Issue
Block a user