update
This commit is contained in:
parent
f954c53980
commit
22d5c9c503
@ -6,9 +6,8 @@ sudo ./build.sh
|
||||
```
|
||||
|
||||
hdfs路径：
|
||||
http://172.17.0.2:50070/explorer.html#/
|
||||
http://172.17.0.2:9870/dfshealth.html#tab-overview
|
||||
|
||||
http://172.17.0.2:50075/datanode.html
|
||||
|
||||
yarn路径
|
||||
http://172.17.0.2:8088/cluster
|
||||
|
||||
@ -7,5 +7,6 @@ docker stop hadoop
|
||||
docker rm hadoop
|
||||
|
||||
docker run -dit --name hadoop --privileged=true \
|
||||
-v /data/hadoop/dfs:/dfs/data \
|
||||
-p 8042 -p 8088 -p 19888 -p 50070 -p 50075 zeekling/hadoop
|
||||
|
||||
|
||||
14
hadoop/hadoop/.dockerignore
Normal file
14
hadoop/hadoop/.dockerignore
Normal file
@ -0,0 +1,14 @@
|
||||
*.bak
|
||||
*.md
|
||||
*.orig
|
||||
*.swp
|
||||
*.tmp
|
||||
.*
|
||||
.DS_Store
|
||||
.aws
|
||||
.env
|
||||
.gitignore
|
||||
Dockerfile
|
||||
LICENSE
|
||||
Makefile
|
||||
docker-compose.yml
|
||||
52
hadoop/hadoop/CentOS-8-reg.repo
Normal file
52
hadoop/hadoop/CentOS-8-reg.repo
Normal file
@ -0,0 +1,52 @@
|
||||
# CentOS-Base.repo
|
||||
#
|
||||
# The mirror system uses the connecting IP address of the client and the
|
||||
# update status of each mirror to pick mirrors that are updated to and
|
||||
# geographically close to the client. You should use this for CentOS updates
|
||||
# unless you are manually picking other mirrors.
|
||||
#
|
||||
# If the mirrorlist= does not work for you, as a fall back you can try the
|
||||
# remarked out baseurl= line instead.
|
||||
#
|
||||
#
|
||||
|
||||
[BaseOS]
|
||||
name=CentOS-$releasever - Base - repo.huaweicloud.com
|
||||
baseurl=https://repo.huaweicloud.com/centos-vault/8.5.2111/BaseOS/$basearch/os/
|
||||
#mirrorlist=https://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=BaseOS&infra=$infra
|
||||
gpgcheck=1
|
||||
gpgkey=https://repo.huaweicloud.com/centos/RPM-GPG-KEY-CentOS-Official
|
||||
|
||||
#released updates
|
||||
[AppStream]
|
||||
name=CentOS-$releasever - AppStream - repo.huaweicloud.com
|
||||
baseurl=https://repo.huaweicloud.com/centos-vault/8.5.2111/AppStream/$basearch/os/
|
||||
#mirrorlist=https://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=AppStream&infra=$infra
|
||||
gpgcheck=1
|
||||
gpgkey=https://repo.huaweicloud.com/centos/RPM-GPG-KEY-CentOS-Official
|
||||
|
||||
[PowerTools]
|
||||
name=CentOS-$releasever - PowerTools - repo.huaweicloud.com
|
||||
baseurl=https://repo.huaweicloud.com/centos-vault/8.5.2111/PowerTools/$basearch/os/
|
||||
#mirrorlist=https://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=PowerTools&infra=$infra
|
||||
gpgcheck=1
|
||||
gpgkey=https://repo.huaweicloud.com/centos/RPM-GPG-KEY-CentOS-Official
|
||||
|
||||
|
||||
#additional packages that may be useful
|
||||
[extras]
|
||||
name=CentOS-$releasever - Extras - repo.huaweicloud.com
|
||||
baseurl=https://repo.huaweicloud.com/centos-vault/8.5.2111/extras/$basearch/os/
|
||||
#mirrorlist=https://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=extras
|
||||
gpgcheck=1
|
||||
gpgkey=https://repo.huaweicloud.com/centos/RPM-GPG-KEY-CentOS-Official
|
||||
|
||||
|
||||
#additional packages that extend functionality of existing packages
|
||||
[centosplus]
|
||||
name=CentOS-$releasever - Plus - repo.huaweicloud.com
|
||||
baseurl=https://repo.huaweicloud.com/centos-vault/8.5.2111/centosplus/$basearch/os/
|
||||
#mirrorlist=https://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=centosplus
|
||||
gpgcheck=1
|
||||
enabled=0
|
||||
gpgkey=https://repo.huaweicloud.com/centos/RPM-GPG-KEY-CentOS-Official
|
||||
99
hadoop/hadoop/Dockerfile
Normal file
99
hadoop/hadoop/Dockerfile
Normal file
@ -0,0 +1,99 @@
|
||||
#
|
||||
# Author: Hari Sekhon
|
||||
# Date: 2016-04-24 21:18:57 +0100 (Sun, 24 Apr 2016)
|
||||
#
|
||||
# vim:ts=4:sts=4:sw=4:et
|
||||
#
|
||||
# https://github.com/HariSekhon/Dockerfiles
|
||||
#
|
||||
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
|
||||
#
|
||||
# https://www.linkedin.com/in/HariSekhon
|
||||
#
|
||||
|
||||
# nosemgrep: dockerfile.audit.dockerfile-source-not-pinned.dockerfile-source-not-pinned
|
||||
FROM harisekhon/centos-java:latest
|
||||
|
||||
ARG HADOOP_VERSION=3.3.1
|
||||
|
||||
LABEL org.opencontainers.image.description="Hadoop" \
|
||||
org.opencontainers.image.version="$HADOOP_VERSION" \
|
||||
org.opencontainers.image.authors="Hari Sekhon (https://www.linkedin.com/in/HariSekhon)" \
|
||||
org.opencontainers.image.url="https://ghcr.io/HariSekhon/hadoop" \
|
||||
org.opencontainers.image.documentation="https://hub.docker.com/r/harisekhon/hadoop" \
|
||||
org.opencontainers.image.source="https://github.com/HariSekhon/Dockerfiles"
|
||||
|
||||
ARG TAR=hadoop-$HADOOP_VERSION.tar.gz
|
||||
|
||||
ENV PATH $PATH:/hadoop/bin
|
||||
|
||||
WORKDIR /
|
||||
|
||||
RUN rm -rf /etc/yum.repos.d/*.repo
|
||||
|
||||
COPY ./CentOS-8-reg.repo /etc/yum.repos.d/CentOS-Base.repo
|
||||
|
||||
RUN yum clean all && yum makecache && yum install -y openssh-server openssh-clients tar which
|
||||
|
||||
RUN set -eux && \
|
||||
yum install -y wget hostname && \
|
||||
# --max-redirect - some apache mirrors redirect a couple times and give you the latest version instead
|
||||
# but this breaks stuff later because the link will not point to the right dir
|
||||
# (and is also the wrong version for the tag)
|
||||
# wget -t 10 --max-redirect 1 --retry-connrefused -O "$TAR" "http://www.apache.org/dyn/closer.lua?filename=hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-$HADOOP_VERSION.tar.gz&action=download" || \
|
||||
wget -t 10 --max-redirect 1 --retry-connrefused -O "$TAR" "http://mirrors.tuna.tsinghua.edu.cn/apache/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-$HADOOP_VERSION.tar.gz" && \
|
||||
tar zxf "$TAR" && \
|
||||
# check tarball was extracted to the right place, helps ensure it's the right version and the link will work
|
||||
test -d "hadoop-$HADOOP_VERSION" && \
|
||||
ln -sv "hadoop-$HADOOP_VERSION" hadoop && \
|
||||
mkdir /etc/hadoop && \
|
||||
ln -s /hadoop/etc/hadoop /etc/hadoop/conf && \
|
||||
rm -fv "$TAR" && \
|
||||
{ rm -rf hadoop/share/doc; : ; } && \
|
||||
yum autoremove -y && \
|
||||
# gets autoremoved, ensure it's added back as Hadoop scripts need it
|
||||
yum install -y hostname && \
|
||||
yum clean all && \
|
||||
rm -rf /var/cache/yum
|
||||
|
||||
COPY entrypoint.sh /
|
||||
COPY conf/core-site.xml /hadoop/etc/hadoop/
|
||||
COPY conf/hdfs-site.xml /hadoop/etc/hadoop/
|
||||
COPY conf/yarn-site.xml /hadoop/etc/hadoop/
|
||||
COPY conf/mapred-site.xml /hadoop/etc/hadoop/
|
||||
COPY profile.d/hadoop.sh /etc/profile.d/
|
||||
COPY ssh/config /root/.ssh/
|
||||
|
||||
RUN set -eux && \
|
||||
# Hadoop 1.x
|
||||
#/hadoop/bin/hadoop namenode -format && \
|
||||
# Hadoop 2.x
|
||||
/hadoop/bin/hdfs namenode -format && \
|
||||
groupadd hadoop && \
|
||||
useradd -g hadoop hdfs && \
|
||||
useradd -g hadoop yarn && \
|
||||
mkdir -p /dfs/name && \
|
||||
mkdir -p /hadoop/logs && \
|
||||
chown -R hdfs:hadoop /dfs/name && \
|
||||
chgrp -R hadoop /hadoop/logs && \
|
||||
chmod -R 0770 /hadoop/logs && \
|
||||
mkdir -p /root/.ssh \
|
||||
/home/hdfs/.ssh \
|
||||
/home/yarn/.ssh && \
|
||||
chown hdfs /home/hdfs/.ssh && \
|
||||
chown yarn /home/yarn/.ssh && \
|
||||
chmod 0700 /root/.ssh \
|
||||
/home/hdfs/.ssh \
|
||||
/home/yarn/.ssh
|
||||
|
||||
ENV HDFS_NAMENODE_USER=hdfs
|
||||
ENV HDFS_SECONDARYNAMENODE_USER=hdfs
|
||||
ENV HDFS_DATANODE_USER=hdfs
|
||||
ENV YARN_RESOURCEMANAGER_USER=yarn
|
||||
ENV YARN_NODEMANAGER_USER=yarn
|
||||
|
||||
#EXPOSE 8020 8042 8088 9000 10020 19888 50010 50020 50070 50075 50090
|
||||
# Hadoop 3.0 changed ports :-(
|
||||
EXPOSE 8020 8042 8088 9000 9868 9870 10020 19888 50010 50020 50090
|
||||
|
||||
CMD ["/entrypoint.sh"]
|
||||
18
hadoop/hadoop/Makefile
Normal file
18
hadoop/hadoop/Makefile
Normal file
@ -0,0 +1,18 @@
|
||||
#
|
||||
# Author: Hari Sekhon
|
||||
# Date: 2016-04-24 21:32:37 +0100 (Sun, 24 Apr 2016)
|
||||
#
|
||||
# vim:ts=4:sts=4:sw=4:noet
|
||||
#
|
||||
# https://github.com/HariSekhon/Dockerfiles
|
||||
#
|
||||
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help improve or steer this or other code I publish
|
||||
#
|
||||
# https://www.linkedin.com/in/HariSekhon
|
||||
#
|
||||
|
||||
REPO := harisekhon/hadoop
|
||||
|
||||
include ../Makefile.in
|
||||
|
||||
MAP_PORTS := 8042 8088 19888 50010 50020 50070 50075 50090
|
||||
43
hadoop/hadoop/README.md
Normal file
43
hadoop/hadoop/README.md
Normal file
@ -0,0 +1,43 @@
|
||||
# Apache Hadoop
|
||||
|
||||
[](https://hub.docker.com/repository/docker/harisekhon/hadoop)
|
||||
|
||||
https://hadoop.apache.org/
|
||||
|
||||
Big Data Distributed Storage and Compute Software
|
||||
|
||||
- Yarn - Distributed Processing Framework for running MapReduce, Spark and other application frameworks
|
||||
- HDFS - Distributed Storage
|
||||
|
||||
By default starts a pseudo-distributed cluster of 4 daemons in a single container:
|
||||
|
||||
- Yarn
|
||||
- ResourceManager - Cluster Processing Master (submit jobs here)
|
||||
- NodeManager - Cluster Processing Worker
|
||||
- HDFS
|
||||
- NameNode - Filesystem Master
|
||||
- DataNode - Filesystem Worker
|
||||
|
||||
Perfect for development and testing. Recommended to use Docker with 4GB+ RAM for this pseudo-cluster container.
|
||||
|
||||
For real scaling just start a single daemon in each container for fully distributed setup.
|
||||
|
||||
|
||||
To run the all-in-one-container cluster and expose all the UIs for NodeManager, ResourceManager, NameNode and DataNode respectively, do:
|
||||
```
|
||||
docker run -ti -p 8042 -p 8088 -p 19888 -p 50070 -p 50075 harisekhon/hadoop
|
||||
```
|
||||
|
||||
or with docker-compose:
|
||||
|
||||
```
|
||||
docker-compose up
|
||||
```
|
||||
|
||||
or without `docker-compose`, a shortcut for the docker run command:
|
||||
|
||||
```
|
||||
make run
|
||||
```
|
||||
|
||||
Related Docker images can be found for many Open Source, Big Data and NoSQL technologies on [my DockerHub profile](https://hub.docker.com/r/harisekhon). The source for them all can be found in the [master Dockerfiles GitHub repo](https://github.com/HariSekhon/Dockerfiles/).
|
||||
8
hadoop/hadoop/conf/core-site.xml
Normal file
8
hadoop/hadoop/conf/core-site.xml
Normal file
@ -0,0 +1,8 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
|
||||
<configuration>
|
||||
<property>
|
||||
<name>fs.defaultFS</name>
|
||||
<value>hdfs://localhost:8020</value>
|
||||
</property>
|
||||
</configuration>
|
||||
16
hadoop/hadoop/conf/hdfs-site.xml
Normal file
16
hadoop/hadoop/conf/hdfs-site.xml
Normal file
@ -0,0 +1,16 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
|
||||
<configuration>
|
||||
<property>
|
||||
<name>dfs.replication</name>
|
||||
<value>1</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.namenode.name.dir</name>
|
||||
<value>/dfs/name</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>dfs.datanode.data.dir</name>
|
||||
<value>/dfs/data</value>
|
||||
</property>
|
||||
</configuration>
|
||||
8
hadoop/hadoop/conf/mapred-site.xml
Normal file
8
hadoop/hadoop/conf/mapred-site.xml
Normal file
@ -0,0 +1,8 @@
|
||||
<?xml version="1.0"?>
|
||||
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
|
||||
<configuration>
|
||||
<property>
|
||||
<name>mapreduce.framework.name</name>
|
||||
<value>yarn</value>
|
||||
</property>
|
||||
</configuration>
|
||||
20
hadoop/hadoop/conf/yarn-site.xml
Normal file
20
hadoop/hadoop/conf/yarn-site.xml
Normal file
@ -0,0 +1,20 @@
|
||||
<?xml version="1.0"?>
|
||||
<configuration>
|
||||
<property>
|
||||
<name>yarn.nodemanager.aux-services</name>
|
||||
<value>mapreduce_shuffle</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage</name>
|
||||
<value>100</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>yarn.nodemanager.resource.cpu-vcores</name>
|
||||
<value>1</value>
|
||||
</property>
|
||||
<!-- needs to be set to 3GB in order for MapReduce MonteCarlo default job to succeed, see tests/test_hadoop.sh in Advanced Nagios Plugins Collection which does extensive tests against harisekhon/hadoop-dev image -->
|
||||
<property>
|
||||
<name>yarn.nodemanager.resource.memory-mb</name>
|
||||
<value>3072</value>
|
||||
</property>
|
||||
</configuration>
|
||||
32
hadoop/hadoop/docker-compose.yml
Normal file
32
hadoop/hadoop/docker-compose.yml
Normal file
@ -0,0 +1,32 @@
|
||||
# vim:ts=2:sts=2:sw=2:et
|
||||
#
|
||||
# Author: Hari Sekhon
|
||||
# Date: 2016-12-09 21:25:07 +0000 (Fri, 09 Dec 2016)
|
||||
#
|
||||
# https://github.com/HariSekhon/Dockerfiles
|
||||
#
|
||||
# License: see accompanying Hari Sekhon LICENSE file
|
||||
#
|
||||
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
|
||||
#
|
||||
# https://www.linkedin.com/in/HariSekhon
|
||||
#
|
||||
|
||||
version: '3'
|
||||
services:
|
||||
hadoop:
|
||||
image: harisekhon/hadoop:${VERSION:-latest}
|
||||
ports:
|
||||
#- 8020:8020
|
||||
- 8042:8042
|
||||
- 8088:8088
|
||||
#- 9000:9000
|
||||
- 9864:9864
|
||||
- 9870:9870
|
||||
#- 10020:10020
|
||||
- 19888:19888
|
||||
- 50010:50010
|
||||
- 50020:50020
|
||||
- 50070:50070
|
||||
- 50075:50075
|
||||
- 50090:50090
|
||||
88
hadoop/hadoop/entrypoint.sh
Executable file
88
hadoop/hadoop/entrypoint.sh
Executable file
@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env bash
|
||||
# vim:ts=4:sts=4:sw=4:et
|
||||
#
|
||||
# Author: Hari Sekhon
|
||||
# Date: 2016-04-24 21:29:46 +0100 (Sun, 24 Apr 2016)
|
||||
#
|
||||
# https://github.com/HariSekhon/Dockerfiles
|
||||
#
|
||||
# License: see accompanying Hari Sekhon LICENSE file
|
||||
#
|
||||
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
|
||||
#
|
||||
# https://www.linkedin.com/in/HariSekhon
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
[ -n "${DEBUG:-}" ] && set -x
|
||||
|
||||
export JAVA_HOME="${JAVA_HOME:-/usr}"
|
||||
|
||||
export PATH="$PATH:/hadoop/sbin:/hadoop/bin"
|
||||
|
||||
if [ $# -gt 0 ]; then
|
||||
exec "$@"
|
||||
else
|
||||
for x in root hdfs yarn; do
|
||||
if ! [ -f "/$x/.ssh/id_rsa" ]; then
|
||||
su - "$x" <<-EOF
|
||||
[ -n "${DEBUG:-}" ] && set -x
|
||||
ssh-keygen -t rsa -f ~/.ssh/id_rsa -N ""
|
||||
EOF
|
||||
fi
|
||||
if ! [ -f "$x/.ssh/authorized_keys" ]; then
|
||||
su - "$x" <<-EOF
|
||||
[ -n "${DEBUG:-}" ] && set -x
|
||||
cp -rfv ~/.ssh/id_rsa.pub ~/.ssh/authorized_keys
|
||||
chmod -v 0400 ~/.ssh/authorized_keys
|
||||
EOF
|
||||
fi
|
||||
done
|
||||
|
||||
# removed in newer versions of CentOS
|
||||
if ! [ -f /etc/ssh/ssh_host_rsa_key ] && [ -x /usr/sbin/sshd-keygen ]; then
|
||||
/usr/sbin/sshd-keygen || :
|
||||
fi
|
||||
if ! [ -f /etc/ssh/ssh_host_rsa_key ]; then
|
||||
ssh-keygen -q -t rsa -f /etc/ssh/ssh_host_rsa_key -C '' -N ''
|
||||
chmod 0600 /etc/ssh/ssh_host_rsa_key
|
||||
chmod 0644 /etc/ssh/ssh_host_rsa_key.pub
|
||||
fi
|
||||
|
||||
if ! pgrep -x sshd &>/dev/null; then
|
||||
/usr/sbin/sshd
|
||||
fi
|
||||
echo
|
||||
SECONDS=0
|
||||
while true; do
|
||||
if ssh-keyscan localhost 2>&1 | grep -q OpenSSH; then
|
||||
echo "SSH is ready to rock"
|
||||
break
|
||||
fi
|
||||
if [ "$SECONDS" -gt 20 ]; then
|
||||
echo "FAILED: SSH failed to come up after 20 secs"
|
||||
exit 1
|
||||
fi
|
||||
echo "waiting for SSH to come up"
|
||||
sleep 1
|
||||
done
|
||||
echo
|
||||
if ! [ -f /root/.ssh/known_hosts ]; then
|
||||
ssh-keyscan localhost || :
|
||||
ssh-keyscan 0.0.0.0 || :
|
||||
fi | tee -a /root/.ssh/known_hosts
|
||||
hostname="$(hostname -f)"
|
||||
if ! grep -q "$hostname" /root/.ssh/known_hosts; then
|
||||
ssh-keyscan "$hostname" || :
|
||||
fi | tee -a /root/.ssh/known_hosts
|
||||
|
||||
mkdir -pv /hadoop/logs
|
||||
|
||||
sed -i "s/localhost/$hostname/" /hadoop/etc/hadoop/core-site.xml
|
||||
rm /run/nologin
|
||||
start-dfs.sh
|
||||
start-yarn.sh
|
||||
tail -f /dev/null /hadoop/logs/*
|
||||
stop-yarn.sh
|
||||
stop-dfs.sh
|
||||
fi
|
||||
29
hadoop/hadoop/get_versions
Executable file
29
hadoop/hadoop/get_versions
Executable file
@ -0,0 +1,29 @@
|
||||
#!/usr/bin/env bash
|
||||
# vim:ts=4:sts=4:sw=4:et
|
||||
#
|
||||
# Author: Hari Sekhon
|
||||
# Date: 2018-10-07 20:46:38 +0100 (Sun, 07 Oct 2018)
|
||||
#
|
||||
# https://github.com/HariSekhon/Dockerfiles
|
||||
#
|
||||
# License: see accompanying Hari Sekhon LICENSE file
|
||||
#
|
||||
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
|
||||
#
|
||||
# https://www.linkedin.com/in/HariSekhon
|
||||
#
|
||||
|
||||
# Get list of versions to be used by tests/check_for_new_version
|
||||
|
||||
set -euo pipefail
|
||||
[ -n "${DEBUG:-}" ] && set -x
|
||||
|
||||
get_versions(){
|
||||
curl -sS http://archive.apache.org/dist/hadoop/common/ |
|
||||
# no grep -P on Mac, requires GNU grep :-/
|
||||
egrep -i -o 'href="hadoop-[[:digit:]]+(\.[[:digit:]]+)+(-[[:alpha:]]+)?' |
|
||||
grep -v -e alpha -e beta |
|
||||
sed 's/href="hadoop-//'
|
||||
}
|
||||
|
||||
get_versions
|
||||
132
hadoop/hadoop/hooks/post_build
Executable file
132
hadoop/hadoop/hooks/post_build
Executable file
@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env bash
|
||||
# vim:ts=4:sts=4:sw=4:et
|
||||
#
|
||||
# Author: Hari Sekhon
|
||||
# Date: 2016-12-17 19:47:36 +0000 (Sat, 17 Dec 2016)
|
||||
#
|
||||
# https://github.com/HariSekhon/Dockerfiles
|
||||
#
|
||||
# License: see accompanying Hari Sekhon LICENSE file
|
||||
#
|
||||
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
|
||||
#
|
||||
# https://www.linkedin.com/in/HariSekhon
|
||||
#
|
||||
|
||||
set -eu
|
||||
[ -n "${DEBUG:-}" ] && set -x
|
||||
srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
highest_tag="unknown"
|
||||
|
||||
current_scala="2.11"
|
||||
|
||||
if [ -z "${IMAGE_NAME:-}" ]; then
|
||||
echo "IMAGE_NAME not set, determining from Makefile"
|
||||
IMAGE_NAME="$(awk -F= '/^[[:space:]]*REPO[[:space:]]*:*=/{print $2}' "$srcdir/../Makefile" | sed 's/[[:space:]]//g;s/"//g')"
|
||||
if [ -z "$IMAGE_NAME" ]; then
|
||||
echo "FAILED to determine IMAGE_NAME From Makefile"
|
||||
exit 1
|
||||
fi
|
||||
echo "determined image name from Makefile to be '$IMAGE_NAME'"
|
||||
fi
|
||||
DOCKER_REPO="${DOCKER_REPO:-${IMAGE_NAME/:*}}"
|
||||
|
||||
if [ -z "${SOURCE_BRANCH:-}" ]; then
|
||||
echo "SOURCE_BRANCH not set, determining from Makefile"
|
||||
SOURCE_BRANCH="$(git branch | sed -n 's/*[[:space:]]*//p')"
|
||||
echo "determined source branch from git to be '$SOURCE_BRANCH'"
|
||||
fi
|
||||
# readlink is not portable across Linux + Mac
|
||||
docker_branch_base="$(basename "$(cd "$(dirname "${BASH_SOURCE[0]}")/.."; echo "$PWD")")"
|
||||
if ! git branch -a | grep -q "$docker_branch_base"; then
|
||||
docker_branch_base="${docker_branch_base%-dev}"
|
||||
docker_branch_base="${docker_branch_base%-cluster}"
|
||||
if ! git branch -a | grep -q "$docker_branch_base"; then
|
||||
docker_branch_base="${docker_branch_base%-*}"
|
||||
fi
|
||||
# solrcloud -> solr
|
||||
docker_branch_base="${docker_branch_base%cloud}"
|
||||
if ! git branch -a | grep -q "$docker_branch_base"; then
|
||||
if [ "$SOURCE_BRANCH" = "master" ]; then
|
||||
echo "WARNING: cannot determine if docker branch base '$docker_branch_base' is correct while running on master branch (DockerHub only checks out master branch)"
|
||||
else
|
||||
echo "ERROR: failed to determine docker branch base even after normalization rules, tried '$docker_branch_base'"
|
||||
echo
|
||||
echo "These are the branches found:"
|
||||
echo
|
||||
git branch -a
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
echo "determined docker branch base = $docker_branch_base"
|
||||
docker_version_base="${docker_branch_base//-/_}"
|
||||
echo "will search Dockerfile using docker version base = $docker_version_base"
|
||||
if [ "$docker_branch_base" = "kafka" ]; then
|
||||
highest_tag="${current_scala}_0.9"
|
||||
echo "Kafka detected, overriding highest tag to be $highest_tag"
|
||||
else
|
||||
# This depends on the Dockerfiles content and version ordering being correct, but this also causes a lot of merge conflicts when merging improvements from master so not storing all previous versions commented out any more as it's more of an inconvenience
|
||||
#highest_tag="$(grep -i ARG.*${docker_version_base}_VERSION= Dockerfile | sed -n '$s/.*=//p' | awk -F. '{print $1"."$2}')"
|
||||
# use Git branch versions as the source of truth
|
||||
branches="$(git branch -a | grep -i -e "[[:space:]]$docker_branch_base\\>" -e "remotes/.*/$docker_branch_base\\>" || :)"
|
||||
if [ -n "$branches" ]; then
|
||||
#if [ "$(wc -l <<< "$branches")" -ge 2 ]; then
|
||||
highest_tag="$(sed 's/.*\/// ; s/.*-\([[:digit:]]\)/\1/' <<< "$branches" | sed 's/\./ /g' | sort -n -k1 -k2 | sed 's/ /./g' | tail -n 1)"
|
||||
echo "determined highest tag from git branches to be '$highest_tag'"
|
||||
#else
|
||||
# echo "WARNING: less than 2 matching branches found, cannot determine the highest tag"
|
||||
#fi
|
||||
fi
|
||||
fi
|
||||
|
||||
current_tag=""
|
||||
if ! grep ':' <<< "$IMAGE_NAME"; then
|
||||
echo "current tag not found in IMAGE_NAME=$IMAGE_NAME, determining from Dockerfile"
|
||||
current_tag="$(grep -i "^[[:space:]]*ARG.*${docker_version_base}.*_VERSION=" "$srcdir/../Dockerfile" | sed -n '$s/.*=//;s/[[:space:]]*//g;s/"//g;p' || :)"
|
||||
if [ -z "$current_tag" ]; then
|
||||
echo "ERROR: failed to determine current tag from Dockerfile!"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$current_tag" != "latest" ]; then
|
||||
current_tag="$(awk -F. '{print $1"."$2}' <<< "$current_tag")"
|
||||
fi
|
||||
if [ "$docker_branch_base" = "kafka" ]; then
|
||||
current_tag="${current_scala}_$current_tag"
|
||||
fi
|
||||
echo "current tag determined from Dockerfile to be '$current_tag'"
|
||||
IMAGE_NAME="$IMAGE_NAME:$current_tag"
|
||||
fi
|
||||
if [ -z "$current_tag" ]; then
|
||||
echo "WARNING: failed to determine current tag - IMAGE_NAME '$IMAGE_NAME' did not contain version?"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
is_latest(){
|
||||
if [ "$SOURCE_BRANCH" = "master" ]; then
|
||||
echo "determined to be latest build, due to running on master branch"
|
||||
return 0
|
||||
elif [ "$SOURCE_BRANCH" = "$docker_branch_base-$highest_tag" ]; then
|
||||
echo "determined to be highest tagged build, due to SOURCE_BRANCH = '$SOURCE_BRANCH' == \$docker_branch_base-\$highest_tag = '$docker_branch_base-$highest_tag'"
|
||||
return 0
|
||||
fi
|
||||
echo "$SOURCE_BRANCH is not the latest build, which is $docker_branch_base-$highest_tag"
|
||||
return 1
|
||||
}
|
||||
|
||||
if is_latest; then
|
||||
if [ "$current_tag" = "latest" ] &&
|
||||
[ -x "$srcdir/../get_latest_version.sh" ]; then
|
||||
newest_version="$("$srcdir/../get_latest_version.sh")"
|
||||
echo "docker tag \"$IMAGE_NAME\" \"$DOCKER_REPO:$newest_version\""
|
||||
docker tag "$IMAGE_NAME" "$DOCKER_REPO:$newest_version"
|
||||
echo "docker push \"$DOCKER_REPO:$newest_version\""
|
||||
docker push "$DOCKER_REPO:$newest_version"
|
||||
else
|
||||
echo "docker tag \"$IMAGE_NAME\" \"$DOCKER_REPO:latest\""
|
||||
docker tag "$IMAGE_NAME" "$DOCKER_REPO:latest"
|
||||
echo "docker push \"$DOCKER_REPO:latest\""
|
||||
docker push "$DOCKER_REPO:latest"
|
||||
fi
|
||||
fi
|
||||
3
hadoop/hadoop/profile.d/hadoop.sh
Normal file
3
hadoop/hadoop/profile.d/hadoop.sh
Normal file
@ -0,0 +1,3 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
export HADOOP_HOME=/hadoop
|
||||
2
hadoop/hadoop/ssh/config
Normal file
2
hadoop/hadoop/ssh/config
Normal file
@ -0,0 +1,2 @@
|
||||
Host *
|
||||
StrictHostKeyChecking no
|
||||
Loading…
Reference in New Issue
Block a user