big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-14816. Update Dockerfile to use Xenial. Contributed by Allen Wittenauer

Browse Source
This commit is contained in:
Chris Douglas 2017-10-19 16:33:47 -07:00
parent cbd2b73ef8
commit ca8ddc6aa4
2 changed files with 98 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

161
dev-support/docker/Dockerfile
View File

@ -18,21 +18,28 @@
# Dockerfile for installing the necessary dependencies for building Hadoop. # Dockerfile for installing the necessary dependencies for building Hadoop.
# See BUILDING.txt. # See BUILDING.txt.
FROM ubuntu:xenial
FROM ubuntu:trusty
WORKDIR /root WORKDIR /root
#####
# Disable suggests/recommends
#####
RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras
RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras
ENV DEBIAN_FRONTEND noninteractive ENV DEBIAN_FRONTEND noninteractive
ENV DEBCONF_TERSE true ENV DEBCONF_TERSE true
###### ######
# Install common dependencies from packages # Install common dependencies from packages. Versions here are either
# sufficient or irrelevant.
# #
# WARNING: DO NOT PUT JAVA APPS HERE! Otherwise they will install default # WARNING: DO NOT PUT JAVA APPS HERE! Otherwise they will install default
# Ubuntu Java. See Java section below! # Ubuntu Java. See Java section below!
###### ######
RUN apt-get -q update && apt-get -q install --no-install-recommends -y \ RUN apt-get -q update && apt-get -q install -y \
apt-utils \
build-essential \ build-essential \
bzip2 \ bzip2 \
curl \ curl \
@ -42,7 +49,6 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
gcc \ gcc \
git \ git \
gnupg-agent \ gnupg-agent \
make \
libbz2-dev \ libbz2-dev \
libcurl4-openssl-dev \ libcurl4-openssl-dev \
libfuse-dev \ libfuse-dev \
@ -51,106 +57,110 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
libsnappy-dev \ libsnappy-dev \
libssl-dev \ libssl-dev \
libtool \ libtool \
locales \
make \
pinentry-curses \ pinentry-curses \
pkg-config \ pkg-config \
protobuf-compiler \
protobuf-c-compiler \
python \ python \
python2.7 \ python2.7 \
python2.7-dev \
python-pip \ python-pip \
python-pkg-resources \
python-setuptools \
python-wheel \
rsync \ rsync \
software-properties-common \
snappy \ snappy \
sudo \
zlib1g-dev zlib1g-dev
####### #######
# Oracle Java # OpenJDK 8
####### #######
RUN apt-get -q install -y openjdk-8-jdk
RUN echo "dot_style = mega" > "/root/.wgetrc" #######
RUN echo "quiet = on" >> "/root/.wgetrc" # OpenJDK 9
# w/workaround for
# https://bugs.launchpad.net/ubuntu/+source/openjdk-9/+bug/1593191
#######
RUN apt-get -o Dpkg::Options::="--force-overwrite" \
-q install -y \
openjdk-9-jdk-headless
RUN apt-get -q install --no-install-recommends -y software-properties-common #######
RUN add-apt-repository -y ppa:webupd8team/java # Set default Java
RUN apt-get -q update #######
#
# Auto-accept the Oracle JDK license # By default, OpenJDK sets the default Java to the highest version.
RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections # We want the opposite, soooooo....
RUN apt-get -q install --no-install-recommends -y oracle-java8-installer #
RUN update-java-alternatives --set java-1.8.0-openjdk-amd64
#### RUN update-alternatives --get-selections | grep -i jdk | \
# Apps that require Java while read line; do \
### alternative=$(echo $line | awk '{print $1}'); \
RUN apt-get -q update && apt-get -q install --no-install-recommends -y ant path=$(echo $line | awk '{print $3}'); \
newpath=$(echo $path | sed -e 's/java-9/java-8/'); \
update-alternatives --set $alternative $newpath; \
done
###### ######
# Install Apache Maven # Install cmake 3.1.0 (3.5.1 ships with Xenial)
######
RUN mkdir -p /opt/maven && \
curl -L -s -S \
https://www-us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz \
-o /opt/maven.tar.gz && \
tar xzf /opt/maven.tar.gz --strip-components 1 -C /opt/maven
ENV MAVEN_HOME /opt/maven
ENV PATH "${PATH}:/opt/maven/bin"
######
# Install cmake
###### ######
RUN mkdir -p /opt/cmake && \ RUN mkdir -p /opt/cmake && \
curl -L -s -S \ curl -L -s -S \
https://cmake.org/files/v3.1/cmake-3.1.0-Linux-x86_64.tar.gz \ https://cmake.org/files/v3.1/cmake-3.1.0-Linux-x86_64.tar.gz \
-o /opt/cmake.tar.gz && \ -o /opt/cmake.tar.gz && \
tar xzf /opt/cmake.tar.gz --strip-components 1 -C /opt/cmake tar xzf /opt/cmake.tar.gz --strip-components 1 -C /opt/cmake
ENV CMAKE_HOME /opt/cmake ENV CMAKE_HOME /opt/cmake
ENV PATH "${PATH}:/opt/cmake/bin" ENV PATH "${PATH}:/opt/cmake/bin"
###### ######
# Install findbugs # Install Google Protobuf 2.5.0 (2.6.0 ships with Xenial)
###### ######
RUN mkdir -p /opt/findbugs && \ RUN mkdir -p /opt/protobuf-src && \
curl -L -s -S \ curl -L -s -S \
https://sourceforge.net/projects/findbugs/files/findbugs/3.0.1/findbugs-noUpdateChecks-3.0.1.tar.gz/download \ https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz \
-o /opt/findbugs.tar.gz && \ -o /opt/protobuf.tar.gz && \
tar xzf /opt/findbugs.tar.gz --strip-components 1 -C /opt/findbugs tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src
ENV FINDBUGS_HOME /opt/findbugs RUN cd /opt/protobuf-src && ./configure --prefix=/opt/protobuf && make install
ENV PATH "${PATH}:/opt/findbugs/bin" ENV PROTOBUF_HOME /opt/protobuf
ENV PATH "${PATH}:/opt/protobuf/bin"
######
# Install Apache Maven 3.3.9 (3.3.9 ships with Xenial)
######
RUN apt-get -q update && apt-get -q install -y maven
ENV MAVEN_HOME /usr
######
# Install findbugs 3.0.1 (3.0.1 ships with Xenial)
######
RUN apt-get -q update && apt-get -q install -y findbugs
ENV FINDBUGS_HOME /usr
#### ####
# Install shellcheck # Install shellcheck (0.4.6, the latest as of 2017-09-26)
#### ####
RUN apt-get -q install -y cabal-install RUN add-apt-repository -y ppa:jonathonf/ghc-8.0.2
RUN mkdir /root/.cabal RUN apt-get -q update && apt-get -q install -y shellcheck
RUN echo "remote-repo: hackage.fpcomplete.com:http://hackage.fpcomplete.com/" >> /root/.cabal/config
#RUN echo "remote-repo: hackage.haskell.org:http://hackage.haskell.org/" > /root/.cabal/config
RUN echo "remote-repo-cache: /root/.cabal/packages" >> /root/.cabal/config
RUN cabal update
RUN cabal install shellcheck --global
#### ####
# Install bats # Install bats (0.4.0, the latest as of 2017-09-26, ships with Xenial)
#### ####
RUN add-apt-repository -y ppa:duggan/bats RUN apt-get -q update && apt-get -q install -y bats
RUN apt-get -q update
RUN apt-get -q install --no-install-recommends -y bats
#### ####
# Install pylint # Install pylint (always want latest)
#### ####
RUN pip install pylint RUN pip2 install pylint
#### ####
# Install dateutil.parser # Install dateutil.parser
#### ####
RUN pip install python-dateutil RUN pip2 install python-dateutil
### ###
# Avoid out of memory errors in builds # Install node.js for web UI framework (4.2.6 ships with Xenial)
###
ENV MAVEN_OPTS -Xms256m -Xmx512m
###
# Install node js tools for web UI frameowkr
### ###
RUN apt-get -y install nodejs && \ RUN apt-get -y install nodejs && \
ln -s /usr/bin/nodejs /usr/bin/node && \ ln -s /usr/bin/nodejs /usr/bin/node && \
@ -159,6 +169,12 @@ RUN apt-get -y install nodejs && \
npm install -g bower && \ npm install -g bower && \
npm install -g ember-cli npm install -g ember-cli
###
# Avoid out of memory errors in builds
###
ENV MAVEN_OPTS -Xms256m -Xmx1g
### ###
# Everything past this point is either not needed for testing or breaks Yetus. # Everything past this point is either not needed for testing or breaks Yetus.
# So tell Yetus not to read the rest of the file: # So tell Yetus not to read the rest of the file:
@ -166,12 +182,17 @@ RUN apt-get -y install nodejs && \
### ###
#### ####
# Install Forrest (for Apache Hadoop website) # Install svn, Ant, & Forrest (for Apache Hadoop website)
### ###
RUN mkdir -p /usr/local/apache-forrest ; \ RUN apt-get -q update && apt-get -q install -y ant subversion
curl -s -S -O http://archive.apache.org/dist/forrest/0.8/apache-forrest-0.8.tar.gz ; \
tar xzf *forrest* --strip-components 1 -C /usr/local/apache-forrest ; \ RUN mkdir -p /opt/apache-forrest && \
echo 'forrest.home=/usr/local/apache-forrest' > build.properties curl -L -s -S \
https://archive.apache.org/dist/forrest/0.8/apache-forrest-0.8.tar.gz \
-o /opt/forrest.tar.gz && \
tar xzf /opt/forrest.tar.gz --strip-components 1 -C /opt/apache-forrest
RUN echo 'forrest.home=/opt/apache-forrest' > build.properties
ENV FORREST_HOME=/opt/apache-forrest
# Add a welcome message and environment checks. # Add a welcome message and environment checks.
ADD hadoop_env_checks.sh /root/hadoop_env_checks.sh ADD hadoop_env_checks.sh /root/hadoop_env_checks.sh

15
dev-support/docker/hadoop_env_checks.sh
View File

@ -16,6 +16,8 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# SHELLDOC-IGNORE
# ------------------------------------------------------- # -------------------------------------------------------
function showWelcome { function showWelcome {
cat <<Welcome-message cat <<Welcome-message
@ -80,14 +82,10 @@ End-of-message
# ------------------------------------------------------- # -------------------------------------------------------
# Configurable low water mark in GiB
MINIMAL_MEMORY_GiB=2
function warnIfLowMemory { function warnIfLowMemory {
MINIMAL_MEMORY=$((MINIMAL_MEMORY_GiB*1024*1024)) # Convert to KiB MINIMAL_MEMORY=2046755
INSTALLED_MEMORY=$(fgrep MemTotal /proc/meminfo | awk '{print $2}') INSTALLED_MEMORY=$(grep -F MemTotal /proc/meminfo | awk '{print $2}')
if [ $((INSTALLED_MEMORY)) -le $((MINIMAL_MEMORY)) ]; if [[ $((INSTALLED_MEMORY)) -lt $((MINIMAL_MEMORY)) ]]; then
then
cat <<End-of-message cat <<End-of-message
_ ___ ___ _ ___ ___
@ -103,7 +101,8 @@ Your system is running on very little memory.
This means it may work but it wil most likely be slower than needed. This means it may work but it wil most likely be slower than needed.
If you are running this via boot2docker you can simply increase If you are running this via boot2docker you can simply increase
the available memory to atleast ${MINIMAL_MEMORY_GiB} GiB (you have $((INSTALLED_MEMORY/(1024*1024))) GiB ) the available memory to at least ${MINIMAL_MEMORY}KiB
(you have ${INSTALLED_MEMORY}KiB )
End-of-message End-of-message
fi fi