big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert "YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)"

Browse Source 
This reverts commit 46d6e00166.
This commit is contained in:
Wangda Tan 2018-12-04 14:13:06 -08:00
parent 9287ab3642
commit 228156cfd1
21 changed files with 0 additions and 2172 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
View File

@ -65,14 +65,6 @@
</includes>
<fileMode>0755</fileMode>
</fileSet>
<fileSet>
<directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation</directory>
<outputDirectory>/share/hadoop/yarn/submarine-installer</outputDirectory>
<includes>
<include>**/*</include>
</includes>
<fileMode>0755</fileMode>
</fileSet>
<fileSet>
<directory>hadoop-yarn/conf</directory>
<outputDirectory>etc/hadoop</outputDirectory>

74
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf
View File

@ -1,74 +0,0 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##### install config #####
# DNS
LOCAL_DNS_HOST="172.17.0.9" # /etc/resolv.conf
YARN_DNS_HOST="10.196.69.173" # yarn dns server ip address
# etcd hosts list
ETCD_HOSTS=(10.196.69.173 10.196.69.174 10.196.69.175)
# docker registry ip:port
DOCKER_REGISTRY="10.120.196.232:5000"
# Start the http download service on the specified server,
# Will download all the dependencies in the http server,
# Run the install script on other servers.
# Automatically download dependencies from http,
# Solve the problem that all servers are slow to download online.
# At the same time, you can also manually download the dependencies to the downloads directory.
# Offline installation of the system
DOWNLOAD_SERVER_IP="10.120.196.236"
DOWNLOAD_SERVER_PORT="19000"
# yarn container-executor config
# How to compile container-executor:
# Go to the hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager
# path and enter the command: mvn package -Pnative -DskipTests
# Only the nodemanager is compiled. The path of the compiled container-executor is:
# ./target/native/target/usr/local/bin/container-executor
YARN_CONTAINER_EXECUTOR_PATH="Please enter the full path to the container-executor here"
# Keep the same configuration as 'yarn.nodemanager.linux-container-executor.cgroups.hierarchy'
# in yarn-site.xml, default '/hadoop-yarn'
YARN_HIERARCHY="/hadoop-yarn"
# Keep the same configuration as 'yarn.nodemanager.local-dirs' in yarn-site.xml
YARN_NODEMANAGER_LOCAL_DIRS="/home/hadoop/disk/1/yarn/local,/home/hadoop/disk/2/yarn/local,/home/hadoop/disk/3/yarn/local"
# Keep the same configuration as 'yarn.nodemanager.log-dirs' in yarn-site.xml
YARN_NODEMANAGER_LOG_DIRS="/home/hadoop/disk/1/yarn/logs,/home/hadoop/disk/2/yarn/logs,/home/hadoop/disk/3/yarn/logs"
##### System component download url address, Generally do not need to be modified #####
DOCKER_REPO="https://yum.dockerproject.org/repo/main/centos/7/Packages"
DOCKER_ENGINE_RPM="docker-engine-1.12.5-1.el7.centos.x86_64.rpm"
DOCKER_ENGINE_SELINUX_RPM="docker-engine-selinux-1.12.5-1.el7.centos.noarch.rpm"
NVIDIA_DETECT_URL="https://raw.githubusercontent.com/liuxunorg/submarine-installer/packages/nvidia/nvidia-detect"
NVIDIA_DOCKER_RPM="nvidia-docker-1.0.1-1.x86_64.rpm"
NVIDIA_DOCKER_RPM_URL="https://github.com/NVIDIA/nvidia-docker/releases/download/v1.0.1/${NVIDIA_DOCKER_RPM}"
ETCD_TAR_GZ="etcd-v3.3.9-linux-amd64.tar.gz"
ETCD_DOWNLOAD_URL="https://github.com/etcd-io/etcd/releases/download/v3.3.9/${ETCD_TAR_GZ}"
CALICO_IPV4POOL_CIDR="192.20.0.0"
CALICO_NETWORK_NAME="calico-network"
CALICO_DOWNLOAD_URL="https://github.com/projectcalico/cni-plugin/releases/download/v1.11.7/calico"
CALICO_IPAM_DOWNLOAD_URL="https://github.com/projectcalico/cni-plugin/releases/download/v1.11.7/calico-ipam"
CALICOCTL_DOWNLOAD_URL="https://github.com/projectcalico/calicoctl/releases/download/v3.2.3/calicoctl"

116
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.sh
View File

@ -1,116 +0,0 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# description: sumbarine install scripts.
ROOT=$(cd "$(dirname "$0")"; pwd)
SUBMARINE_INSTALLER_VERSION="v0.7"
PACKAGE_DIR=${ROOT}/package
SCRIPTS_DIR=${ROOT}/scripts
INSTALL_TEMP_DIR=${ROOT}/temp
DOWNLOAD_DIR=${ROOT}/downloads
DATE=`date +%Y%m%d-%H:%M:%S`
LOG=${ROOT}/logs/install.log.`date +%Y%m%d%H%M%S`
LOCAL_HOST_IP_LIST=()
LOCAL_HOST_IP=""
OPERATING_SYSTEM=""
DOWNLOAD_HTTP=""
# import shell script
. ${ROOT}/install.conf
. ${ROOT}/scripts/calico.sh
. ${ROOT}/scripts/docker.sh
. ${ROOT}/scripts/download-server.sh
. ${ROOT}/scripts/environment.sh
. ${ROOT}/scripts/etcd.sh
. ${ROOT}/scripts/hadoop.sh
. ${ROOT}/scripts/menu.sh
. ${ROOT}/scripts/nvidia.sh
. ${ROOT}/scripts/nvidia-docker.sh
. ${ROOT}/scripts/submarine.sh
. ${ROOT}/scripts/utils.sh
#================================= Main ========================================
mkdir $ROOT/logs/ -p
mkdir $INSTALL_TEMP_DIR -p
mkdir $DOWNLOAD_DIR -p
source /etc/os-release
OPERATING_SYSTEM=$ID
get_ip_list
ipCount=${#LOCAL_HOST_IP_LIST[@]}
if [[ $ipCount -eq 1 ]]; then
LOCAL_HOST_IP=${LOCAL_HOST_IP_LIST[0]}
else
echo -e "Detect the network card IP in the server, \e[31m[${LOCAL_HOST_IP_LIST[@]}]\e[0m"
echo -n -e "please enter a valid IP address: "
read ipInput
if ! valid_ip $ipInput; then
echo -e "you input \e[31m$ipInput\e[0m address format is incorrect! " | tee -a $LOG
exit_install
else
LOCAL_HOST_IP=$ipInput
fi
fi
echo -n -e "Please confirm whether the IP address of this machine is \e[31m${LOCAL_HOST_IP}\e[0m?[y|n]"
read myselect
if [[ "$myselect" != "y" && "$myselect" != "Y" ]]; then
exit_install
fi
check_install_conf
if [[ -n "$DOWNLOAD_SERVER_IP" && -n "$DOWNLOAD_SERVER_PORT" && "$DOWNLOAD_SERVER_IP" != "$LOCAL_HOST_IP" ]]; then
DOWNLOAD_HTTP="http://${DOWNLOAD_SERVER_IP}:${DOWNLOAD_SERVER_PORT}"
fi
check_install_user
# 清理安装临时目录
rm $INSTALL_TEMP_DIR/* -rf >>$LOG 2>&1
menu_index="0"
for ((j=1;;j++))
do
menu
case "$menu_index" in
"0")
menu_index="$menu_choice"
;;
"1"|"2"|"3"|"4"|"5")
# echo "aaaa=$menu_index-$menu_choice"
menu_process
if [[ $? = 1 ]]; then
echo "Press any key to return menu!"
read
fi
;;
"a")
exit_install
;;
"q")
exit_install
;;
*)
menu_index="0"
menu_choice="0"
menu
;;
esac
done

50
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calico-node.service
View File

@ -1,50 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[Unit]
Description=calico node
After=docker.service
Requires=docker.service
[Service]
User=root
PermissionsStartOnly=true
ExecStart=/usr/bin/docker run --net=host --privileged --name=calico-node \
-e ETCD_ENDPOINTS=ETCD_ENDPOINTS_REPLACE \
-e CALICO_LIBNETWORK_ENABLED=true \
-e CALICO_NETWORKING_BACKEND=bird \
-e CALICO_DISABLE_FILE_LOGGING=true \
-e CALICO_IPV4POOL_CIDR=CALICO_IPV4POOL_CIDR_REPLACE/16 \
-e CALICO_IPV4POOL_IPIP=always \
-e FELIX_DEFAULTENDPOINTTOHOSTACTION=ACCEPT \
-e FELIX_IPV6SUPPORT=false \
-e FELIX_LOGSEVERITYSCREEN=info \
-e FELIX_IPINIPMTU=1440 \
-e FELIX_HEALTHENABLED=true \
-e IP= \
-v /etc/calico/ssl:/etc/calico/ssl \
-v /var/run/calico:/var/run/calico \
-v /lib/modules:/lib/modules \
-v /run/docker/plugins:/run/docker/plugins \
-v /var/run/docker.sock:/var/run/docker.sock \
-v /var/log/calico:/var/log/calico \
calico/node:v2.6.2
ExecStop=/usr/bin/docker rm -f calico-node
Restart=always
RestartSec=10
[Install]
WantedBy=multi-user.target

22
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calicoctl.cfg
View File

@ -1,22 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: v1
kind: calicoApiConfig
metadata:
spec:
datastoreType: "etcdv2"
etcdEndpoints: ETCD_ENDPOINTS_REPLACE

23
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/daemon.json
View File

@ -1,23 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
{
"insecure-registries": ["DOCKER_REGISTRY_REPLACE"],
"cluster-store":"CLUSTER_STORE_REPLACE",
"cluster-advertise":"LOCAL_HOST_IP_REPLACE:2375",
"dns": ["YARN_DNS_HOST_REPLACE", "LOCAL_DNS_HOST_REPLACE"],
"hosts": ["tcp://LOCAL_HOST_IP_REPLACE:2375","unix:///var/run/docker.sock"]
}

35
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/docker.service
View File

@ -1,35 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[Unit]
Description=Docker Application Container Engine
Documentation=http://docs.docker.io
[Service]
Environment="PATH=/usr/bin:/bin:/sbin:/usr/bin:/usr/sbin"
ExecStart=/usr/bin/dockerd --log-level=error
ExecStartPost=/sbin/iptables -I FORWARD -s 0.0.0.0/0 -j ACCEPT
ExecReload=/bin/kill -s HUP $MAINPID
Restart=on-failure
RestartSec=5
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
Delegate=yes
KillMode=process
[Install]
WantedBy=multi-user.target

40
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/etcd/etcd.service
View File

@ -1,40 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[Unit]
Description=etcd
Documentation=https://github.com/coreos/etcd
[Service]
Type=notify
Restart=always
RestartSec=5s
LimitNOFILE=40000
TimeoutStartSec=0
ExecStart=/usr/bin/etcd \
--name=ETCD_NODE_NAME_REPLACE \
--data-dir=/var/lib/etcd \
--listen-client-urls=http://LOCAL_HOST_REPLACE:2379,http://127.0.0.1:2379 \
--listen-peer-urls=http://LOCAL_HOST_REPLACE:2380 \
--advertise-client-urls=http://LOCAL_HOST_REPLACE:2379 \
--initial-advertise-peer-urls=http://LOCAL_HOST_REPLACE:2380 \
--initial-cluster=INITIAL_CLUSTER_REPLACE \
--initial-cluster-token=etcd-token \
--initial-cluster-state=new
[Install]
WantedBy=multi-user.target

41
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/hadoop/container-executor.cfg
View File

@ -1,41 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
yarn.nodemanager.local-dirs=YARN_NODEMANAGER_LOCAL_DIRS_REPLACE
yarn.nodemanager.linux-container-executor.group=yarn
yarn.nodemanager.log-dirs=YARN_NODEMANAGER_LOG_DIRS_REPLACE
banned.users=root
allowed.system.users=yarn
min.user.id=500
[docker]
module.enabled=true
docker.binary=/usr/bin/docker
docker.allowed.capabilities=SYS_CHROOT,MKNOD,SETFCAP,SETPCAP,FSETID,CHOWN,AUDIT_WRITE,SETGID,NET_RAW,FOWNER,SETUID,DAC_OVERRIDE,KILL,NET_BIND_SERVICE,DAC_READ_SEARCH,SYS_PTRACE,SYS_ADMIN
docker.allowed.networks=bridge,host,none,CALICO_NETWORK_NAME_REPLACE
docker.allowed.ro-mounts=/etc/group,/etc/passwd,/etc/krb5.conf,YARN_NODEMANAGER_LOCAL_DIRS_REPLACE,regex:^nvidia_driver_.*$
docker.allowed.rw-mounts=YARN_NODEMANAGER_LOCAL_DIRS_REPLACE,YARN_NODEMANAGER_LOG_DIRS_REPLACE
docker.privileged-containers.enabled=false
docker.trusted.registries=local,centos,hortonworks,DOCKER_REGISTRY_REPLACE
docker.allowed.volume-drivers=nvidia-docker
docker.allowed.devices=regex:^/dev/nvidia.*$
[gpu]
module.enabled=true
[cgroups]
root=/sys/fs/cgroup
yarn-hierarchy=YARN_HIERARCHY_REPLACE

25
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/submarine/submarine.sh
View File

@ -1,25 +0,0 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Solution: Grant user yarn the access to /sys/fs/cgroup/cpu,cpuacct,
# which is the subfolder of cgroup mount destination.
chown :yarn -R /sys/fs/cgroup/cpu,cpuacct
chmod g+rwx -R /sys/fs/cgroup/cpu,cpuacct
# If GPUs are used，the access to cgroup devices folder is neede as well
chown :yarn -R /sys/fs/cgroup/devices
chmod g+rwx -R /sys/fs/cgroup/devices

224
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/calico.sh
View File

@ -1,224 +0,0 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
## @description download calico bin
## @audience public
## @stability stable
function download_calico_bin()
{
# submarin http server
if [[ -n "$DOWNLOAD_HTTP" ]]; then
MY_CALICOCTL_DOWNLOAD_URL=${DOWNLOAD_HTTP}/downloads/calico/calicoctl
MY_CALICO_DOWNLOAD_URL=${DOWNLOAD_HTTP}/downloads/calico/calico
MY_CALICO_IPAM_DOWNLOAD_URL=${DOWNLOAD_HTTP}/downloads/calico/calico-ipam
else
MY_CALICOCTL_DOWNLOAD_URL=${CALICOCTL_DOWNLOAD_URL}
MY_CALICO_DOWNLOAD_URL=${CALICO_DOWNLOAD_URL}
MY_CALICO_IPAM_DOWNLOAD_URL=${CALICO_IPAM_DOWNLOAD_URL}
fi
mkdir -p ${DOWNLOAD_DIR}/calico
if [[ -f ${DOWNLOAD_DIR}/calico/calico ]]; then
echo "${DOWNLOAD_DIR}/calico/calico is exist."
else
echo "download ${MY_CALICO_DOWNLOAD_URL} ..."
wget -P ${DOWNLOAD_DIR}/calico ${MY_CALICO_DOWNLOAD_URL}
fi
if [[ -f ${DOWNLOAD_DIR}/calico/calicoctl ]]; then
echo "${DOWNLOAD_DIR}/calico is exist."
else
echo "download ${MY_CALICOCTL_DOWNLOAD_URL} ..."
wget -P ${DOWNLOAD_DIR}/calico ${MY_CALICOCTL_DOWNLOAD_URL}
fi
if [[ -f ${DOWNLOAD_DIR}/calico/calico-ipam ]]; then
echo "${DOWNLOAD_DIR}/calico/calico-ipam is exist."
else
echo "download ${MY_CALICO_IPAM_DOWNLOAD_URL} ..."
wget -P ${DOWNLOAD_DIR}/calico ${MY_CALICO_IPAM_DOWNLOAD_URL}
fi
}
## @description install calico bin
## @audience public
## @stability stable
function install_calico_bin()
{
download_calico_bin
cp -f ${DOWNLOAD_DIR}/calico/calico /usr/bin/calico
cp -f ${DOWNLOAD_DIR}/calico/calicoctl /usr/bin/calicoctl
cp -f ${DOWNLOAD_DIR}/calico/calico-ipam /usr/bin/calico-ipam
chmod +x /usr/bin/calico
chmod +x /usr/bin/calicoctl
chmod +x /usr/bin/calico-ipam
}
## @description install calico config
## @audience public
## @stability stable
function install_calico_config()
{
mkdir -p /etc/calico
cp -rf ${PACKAGE_DIR}/calico ${INSTALL_TEMP_DIR}/
# 1. replace etcdEndpoints
# etcdEndpoints: https://10.196.69.173:2379,https://10.196.69.174:2379,https://10.196.69.175:2379
etcdEndpoints=''
index=0
etcdHostsSize=${#ETCD_HOSTS[@]}
for item in ${ETCD_HOSTS[@]}
do
index=$(($index+1))
etcdEndpoints="${etcdEndpoints}http:\/\/${item}:2379"
if [[ ${index} -lt ${etcdHostsSize} ]]; then
etcdEndpoints=${etcdEndpoints}","
fi
done
# echo "etcdEndpoints=${etcdEndpoints}"
sed -i "s/ETCD_ENDPOINTS_REPLACE/${etcdEndpoints}/g" $INSTALL_TEMP_DIR/calico/calicoctl.cfg >>$LOG
if [[ ! -d /etc/calico ]]; then
mkdir /etc/calico
else
rm -rf /etc/calico/*
fi
cp -f $INSTALL_TEMP_DIR/calico/calicoctl.cfg /etc/calico/calicoctl.cfg
sed -i "s/ETCD_ENDPOINTS_REPLACE/${etcdEndpoints}/g" $INSTALL_TEMP_DIR/calico/calico-node.service >>$LOG
sed -i "s/CALICO_IPV4POOL_CIDR_REPLACE/${CALICO_IPV4POOL_CIDR}/g" $INSTALL_TEMP_DIR/calico/calico-node.service >>$LOG
cp $INSTALL_TEMP_DIR/calico/calico-node.service /etc/systemd/system/ >>$LOG
systemctl daemon-reload
systemctl enable calico-node.service
}
## @description modify kernel network config
## @audience public
## @stability stable
function kernel_network_config()
{
if [ `grep -c "net.ipv4.conf.all.rp_filter=1" /etc/sysctl.conf` -eq '0' ]; then
echo "net.ipv4.conf.all.rp_filter=1" >>/etc/sysctl.conf
fi
if [ `grep -c "net.ipv4.ip_forward=1" /etc/sysctl.conf` -eq '0' ]; then
echo "net.ipv4.ip_forward=1" >>/etc/sysctl.conf
fi
sysctl -p
}
## @description check if the calico-network exist
## @audience public
## @stability stable
function calico_network_exist()
{
local dockerNetwokInfo=`docker network ls --filter NAME=${CALICO_NETWORK_NAME}`
echo ${dockerNetwokInfo} | grep ${CALICO_NETWORK_NAME}
}
## @description verification calico
## @audience public
## @stability stable
function verification_calico()
{
echo " ===== Check if the network between 2 containers can be connected ====="
local claicoNetworkExist=`calico_network_exist`
if [[ "$claicoNetworkExist" = "" ]]; then
echo "Create a calico network"
docker network create --driver calico --ipam-driver calico-ipam ${CALICO_NETWORK_NAME}
else
echo "calico network ${CALICO_NETWORK_NAME} is exist."
fi
local verifyA="verify-calico-network-A"
local verifyAInfo=`containers_exist ${verifyA}`
if [[ -n "$verifyAInfo" ]]; then
echo "Delete existing container ${verifyA}."
docker stop ${verifyA}
docker rm ${verifyA}
fi
echo "Create containers verify-calico-network-A"
docker run --net ${CALICO_NETWORK_NAME} --name ${verifyA} -tid busybox
local verifyB="verify-calico-network-B"
local verifyBInfo=`containers_exist ${verifyB}`
if [[ -n "$verifyBInfo" ]]; then
echo "Delete existing container ${verifyB}."
docker stop ${verifyB}
docker rm ${verifyB}
fi
echo "Create containers verify-calico-network-B"
docker run --net ${CALICO_NETWORK_NAME} --name ${verifyB} -tid busybox
echo -e "\033[33m${verifyA} ping ${verifyB}\033[0m"
docker exec ${verifyA} ping ${verifyB} -c 5
}
## @description install calico
## @audience public
## @stability stable
function install_calico()
{
kernel_network_config
install_calico_bin
install_calico_config
start_calico
verification_calico
}
## @description uninstall calico
## @audience public
## @stability stable
function uninstall_calico()
{
echo "stop calico-node.service"
systemctl stop calico-node.service
echo "rm /usr/bin/calico ..."
rm /usr/bin/calicoctl
rm /usr/bin/calico
rm /usr/bin/calico-ipam
rm -rf /etc/calico/
rm /etc/systemd/system/calico-node.service
systemctl daemon-reload
}
## @description start calico
## @audience public
## @stability stable
function start_calico()
{
systemctl restart calico-node.service
systemctl status calico-node.service
}
## @description stop calico
## @audience public
## @stability stable
function stop_calico()
{
systemctl stop calico-node.service
systemctl status calico-node.service
}

166
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/docker.sh
View File

@ -1,166 +0,0 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
## @description download docker rmp
## @audience public
## @stability stable
function download_docker_rpm()
{
# download http server
if [[ -n "$DOWNLOAD_HTTP" ]]; then
MY_DOCKER_ENGINE_SELINUX_RPM="${DOWNLOAD_HTTP}/downloads/docker/${DOCKER_ENGINE_SELINUX_RPM}"
MY_DOCKER_ENGINE_RPM="${DOWNLOAD_HTTP}/downloads/docker/${DOCKER_ENGINE_RPM}"
else
MY_DOCKER_ENGINE_SELINUX_RPM=${DOCKER_REPO}/${DOCKER_ENGINE_SELINUX_RPM}
MY_DOCKER_ENGINE_RPM=${DOCKER_REPO}/${DOCKER_ENGINE_RPM}
fi
# download docker rpm
if [[ -f ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_SELINUX_RPM} ]]; then
echo "${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_SELINUX_RPM} is exist."
else
echo "download ${MY_DOCKER_ENGINE_SELINUX_RPM} ..."
wget -P ${DOWNLOAD_DIR}/docker/ ${MY_DOCKER_ENGINE_SELINUX_RPM}
fi
if [[ -f ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_RPM} ]]; then
echo "${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_RPM} is exist."
else
echo "download ${MY_DOCKER_ENGINE_RPM} ..."
wget -P ${DOWNLOAD_DIR}/docker/ ${MY_DOCKER_ENGINE_RPM}
fi
}
## @description install docker bin
## @audience public
## @stability stable
function install_docker_bin()
{
download_docker_rpm
yum -y localinstall ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_SELINUX_RPM}
yum -y localinstall ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_RPM}
}
## @description uninstall docker bin
## @audience public
## @stability stable
function uninstall_docker_bin()
{
download_docker_rpm
yum -y remove ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_SELINUX_RPM}
yum -y remove ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_RPM}
}
## @description install docker config
## @audience public
## @stability stable
function install_docker_config()
{
rm -rf ${INSTALL_TEMP_DIR}/docker
cp -rf ${PACKAGE_DIR}/docker ${INSTALL_TEMP_DIR}/
# replace cluster-store
# "cluster-store":"etcd://10.196.69.173:2379,10.196.69.174:2379,10.196.69.175:2379"
# char '/' need to escape '\/'
clusterStore="etcd:\/\/"
index=1
etcdHostsSize=${#ETCD_HOSTS[@]}
for item in ${ETCD_HOSTS[@]}
do
clusterStore="${clusterStore}${item}:2379"
if [[ ${index} -lt ${etcdHostsSize}-1 ]]; then
clusterStore=${clusterStore}","
fi
index=$(($index+1))
done
echo "clusterStore=${clusterStore}"
sed -i "s/CLUSTER_STORE_REPLACE/${clusterStore}/g" $INSTALL_TEMP_DIR/docker/daemon.json >>$LOG
sed -i "s/DOCKER_REGISTRY_REPLACE/${DOCKER_REGISTRY}/g" $INSTALL_TEMP_DIR/docker/daemon.json >>$LOG
sed -i "s/LOCAL_HOST_IP_REPLACE/${LOCAL_HOST_IP}/g" $INSTALL_TEMP_DIR/docker/daemon.json >>$LOG
sed -i "s/YARN_DNS_HOST_REPLACE/${YARN_DNS_HOST}/g" $INSTALL_TEMP_DIR/docker/daemon.json >>$LOG
sed -i "s/LOCAL_DNS_HOST_REPLACE/${LOCAL_DNS_HOST}/g" $INSTALL_TEMP_DIR/docker/daemon.json >>$LOG
# Delete the ASF license comment in the daemon.json file, otherwise it will cause a json format error.
sed -i '1,16d' $INSTALL_TEMP_DIR/docker/daemon.json
if [ ! -d "/etc/docker" ]; then
mkdir /etc/docker
fi
cp $INSTALL_TEMP_DIR/docker/daemon.json /etc/docker/
cp $INSTALL_TEMP_DIR/docker/docker.service /etc/systemd/system/ >>$LOG
}
## @description install docker
## @audience public
## @stability stable
function install_docker()
{
install_docker_bin
install_docker_config
systemctl daemon-reload
systemctl enable docker.service
}
## @description unstall docker
## @audience public
## @stability stable
function uninstall_docker()
{
echo "stop docker service"
systemctl stop docker
echo "remove docker"
uninstall_docker_bin
rm /etc/docker/daemon.json >>$LOG
rm /etc/systemd/system/docker.service >>$LOG
systemctl daemon-reload
}
## @description start docker
## @audience public
## @stability stable
function start_docker()
{
systemctl restart docker
systemctl status docker
docker info
}
## @description stop docker
## @audience public
## @stability stable
function stop_docker()
{
systemctl stop docker
systemctl status docker
}
## @description check if the containers exist
## @audience public
## @stability stable
function containers_exist()
{
local dockerContainersInfo=`docker ps -a --filter NAME=$1`
echo ${dockerContainersInfo} | grep $1
}

42
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/download-server.sh
View File

@ -1,42 +0,0 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
## @description start download server
## @audience public
## @stability stable
function start_download_server()
{
if [[ "$DOWNLOAD_SERVER_IP" != "$LOCAL_HOST_IP" ]]; then
echo -e "\033[31mERROR: Only $DOWNLOAD_SERVER_IP can start the download service.\033[0m"
return 1
fi
echo -e "You can put the install package file in the \033[34m${DOWNLOAD_DIR}\033[0m folder first, Or automatic download."
echo -n "Do you want to start download http server?[y|n]"
read myselect
if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
then
download_etcd_bin
download_calico_bin
download_docker_rpm
download_nvidia_driver
download_nvidia_docker_bin
download_yarn_container_executor
python -m SimpleHTTPServer ${DOWNLOAD_SERVER_PORT}
fi
}

213
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/environment.sh
View File

@ -1,213 +0,0 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
## @description check operation System
## @audience public
## @stability stable
function check_operationSystem()
{
echo -e "The submarine assembly support \033[32m[centos-release-7-3.1611.el7.centos.x86_64]\033[0m or higher operating system version."
case ${OPERATING_SYSTEM} in
centos)
local operationSystemVersion=`rpm --query centos-release`
echo -e "The current operating system version is \e[31m[${operationSystemVersion}]\e[0m" | tee -a $LOG
;;
*)
echo -e "\033[31mWARN: The submarine assembly Unsupported [${OPERATING_SYSTEM}] operating system\033[0m"
;;
esac
}
## @description update operation System Kernel
## @audience public
## @stability stable
function update_operationSystemKernel()
{
echo "If the server is unable to connect to the network, execute the following command yourself:
wget http://vault.centos.org/7.3.1611/os/x86_64/Packages/kernel-headers-3.10.0-514.el7.x86_64.rpm
rpm -ivh kernel-headers-3.10.0-514.el7.x86_64.rpm"
echo -n "Do you want to kernel upgrades?[y|n]"
read myselect
if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
then
echo "Now try to use the yum command for kernel upgrades ..."
yum install kernel-devel-$(uname -r) kernel-headers-$(uname -r)
local kernelVersion=`uname -r`
echo -e "After the upgrade, the operating system kernel version is \e[31m${kernelVersion}\e[0m" | tee -a $LOG
fi
}
## @description check operation system kernel
## @audience public
## @stability stable
function check_operationSystemKernel()
{
case ${OPERATING_SYSTEM} in
centos)
local kernelVersion=`uname -r`
echo -e "Submarine support operating system kernel version is \033[32m 3.10.0-514.el7.x86_64 \033[0m" | tee -a $LOG
echo -e "Current operating system kernel version is \e[31m${kernelVersion}\e[0m" | tee -a $LOG
update_operationSystemKernel
;;
*)
echo -e "\033[31m WARN: The submarine assembly Unsupported operating system [${OPERATING_SYSTEM}] \033[0m"
;;
esac
}
## @description get gcc version
## @audience public
## @stability stable
function get_gcc_version()
{
local gccVersion=`gcc --version`
version=${gccVersion%Copyright*}
echo $version
}
## @description install gcc
## @audience public
## @stability stable
function install_gcc()
{
echo -n "Do you want to install gcc?[y|n]"
read myselect
if [[ "$myselect" = "y" || "$myselect" = "Y" ]]; then
echo "Execute the yum install gcc make g++ command"
yum install gcc make g++
local gccVersion=`gcc --version`
echo -e "After the install, the gcc version is \e[31m${gccVersion}\e[0m" | tee -a $LOG
fi
}
## @description check gcc Version
## @audience public
## @stability stable
function check_gccVersion()
{
local gccVersionInfo=`gcc --version`
local gccVersion=${gccVersionInfo%Copyright*}
if [[ "$gccVersion" = "" ]]; then
echo "The gcc was not installed on the system. Automated installation ..."
install_gcc
else
echo -e "Submarine gcc version need \033[34mgcc (GCC) 4.8.5 20150623 (Red Hat 4.8.5-11)\033[0m or higher."
echo -e "Current gcc version was \033[34m${gccVersion}\033[0m"
fi
}
## @description check GPU
## @audience public
## @stability stable
function check_GPU()
{
gpuInfo=`lspci | grep -i nvidia`
if [[ "$gpuInfo" = "" ]]; then
echo -e "\033[31mERROR: The system did not detect the GPU graphics card.\033[0m"
else
echo -e "\033[32mINFO: The system detect the GPU graphics card.\033[0m"
fi
}
## @description check user group
## @audience public
## @stability stable
function check_userGroup()
{
echo -e "check hadoop user group ..."
echo -e "Hadoop runs the required user [hdfs, mapred, yarn] and groups [hdfs, mapred, yarn, hadoop] installed by ambari."
echo -e "If you are not using ambari for hadoop installation,
then you can add the user and group by root by executing the following command:
\033[34madduser hdfs
adduser mapred
adduser yarn
addgroup hadoop
usermod -aG hdfs,hadoop hdfs
usermod -aG mapred,hadoop mapred
usermod -aG yarn,hadoop yarn
usermod -aG hdfs,hadoop hadoop
groupadd docker
usermod -aG docker yarn
usermod -aG docker hadoop\033[0m\n"
echo -e "check docker user group ..."
# check user group
DOCKER_USER_GROUP='docker'
egrep "^${DOCKER_USER_GROUP}" /etc/group >& /dev/null
if [[ $? -ne 0 ]]; then
echo -e "user group ${DOCKER_USER_GROUP} does not exist, Please execute the following command:"
echo -e "\033[34mgroupadd $DOCKER_USER_GROUP\033[0m"
fi
# check user
USER_GROUP=(yarn hadoop)
for user in ${USER_GROUP[@]}
do
egrep "^${user}" /etc/passwd >& /dev/null
if [[ $? -ne 0 ]]; then
echo -e "User ${user} does not exist, Please execute the following command:"
echo -e "\033[34madduser ${user}\033[0m"
echo -e "\033[34musermod -aG ${DOCKER_USER_GROUP} ${user}\033[0m"
fi
echo -e "Please execute the following command:"
echo -e "\033[34musermod -aG ${DOCKER_USER_GROUP} ${user}\033[0m"
done
}
## @description Some preparatory work for nvidia driver installation
## @audience public
## @stability stable
function prepare_nvidia_environment()
{
echo "prepare nvidia environment ..."
yum -y update
yum install kernel-devel-$(uname -r) kernel-headers-$(uname -r)
yum -y install epel-release
yum -y install dkms
echo -e "\033[34m ===== Please manually execute the following command =====
# 1. Disable nouveau
# Add the content 'rd.driver.blacklist=nouveau nouveau.modeset=0'
# to the 'GRUB_CMDLINE_LINUX' configuration item in the /etc/default/grub file.
root:> vi /etc/default/grub
vi:> GRUB_CMDLINE_LINUX=\"rd.driver.blacklist=nouveau nouveau.modeset=0 ...\"
# 2. Generate configuration
root:> grub2-mkconfig -o /boot/grub2/grub.cfg
# 3. Open (new) /etc/modprobe.d/blacklist.conf, add content 'blacklist nouveau'
root:> vi /etc/modprobe.d/blacklist.conf
vi:> blacklist nouveau
# 4. Update configuration and reboot
root:> mv /boot/initramfs-$(uname -r).img /boot/initramfs-$(uname -r)-nouveau.img
root:> dracut /boot/initramfs-$(uname -r).img $(uname -r)
root:> reboot
\033[0m"
}

152
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/etcd.sh
View File

@ -1,152 +0,0 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
## @description download etcd bin
## @audience public
## @stability stable
function download_etcd_bin()
{
# my download http server
if [[ -n "$DOWNLOAD_HTTP" ]]; then
MY_ETCD_DOWNLOAD_URL="${DOWNLOAD_HTTP}/downloads/etcd/${ETCD_TAR_GZ}"
else
MY_ETCD_DOWNLOAD_URL=${ETCD_DOWNLOAD_URL}
fi
if [[ -f "${DOWNLOAD_DIR}/etcd/${ETCD_TAR_GZ}" ]]; then
echo "${DOWNLOAD_DIR}/etcd/${ETCD_TAR_GZ} is exist."
else
echo "download ${MY_ETCD_DOWNLOAD_URL} ..."
wget -P ${DOWNLOAD_DIR}/etcd ${MY_ETCD_DOWNLOAD_URL}
fi
}
## @description install etcd bin
## @audience public
## @stability stable
function install_etcd_bin()
{
download_etcd_bin
# install etcd bin
mkdir -p ${INSTALL_TEMP_DIR}
rm -rf ${INSTALL_TEMP_DIR}/etcd-*-linux-amd6
tar zxvf ${DOWNLOAD_DIR}/etcd/${ETCD_TAR_GZ} -C ${INSTALL_TEMP_DIR}
cp -f ${INSTALL_TEMP_DIR}/etcd-*-linux-amd64/etcd /usr/bin
cp -f ${INSTALL_TEMP_DIR}/etcd-*-linux-amd64/etcdctl /usr/bin
mkdir -p /var/lib/etcd
chmod -R a+rw /var/lib/etcd
}
## @description install etcd config
## @audience public
## @stability stable
function install_etcd_config()
{
# config etcd.service
rm -rf ${INSTALL_TEMP_DIR}/etcd
cp -rf ${PACKAGE_DIR}/etcd ${INSTALL_TEMP_DIR}/
# 1. Replace name with ETCD_NODE_NAME_REPLACE based on the location of the local IP in $ETCD_HOSTS
indexEtcdList=$(indexByEtcdHosts ${LOCAL_HOST_IP})
# echo ${indexEtcdList}
etcdNodeName="etcdnode${indexEtcdList}"
# echo ${etcdNodeName}
sed -i "s/ETCD_NODE_NAME_REPLACE/${etcdNodeName}/g" $INSTALL_TEMP_DIR/etcd/etcd.service >>$LOG
# 2. Replace local IP address
sed -i "s/LOCAL_HOST_REPLACE/${LOCAL_HOST_IP}/g" $INSTALL_TEMP_DIR/etcd/etcd.service >>$LOG
# 3. Replace the initial-cluster parameter
# --initial-cluster=etcdnode1=http://10.196.69.173:2380,etcdnode2=http://10.196.69.174:2380,etcdnode3=http://10.196.69.175:2380 \
initialCluster=''
index=0
etcdHostsSize=${#ETCD_HOSTS[@]}
for item in ${ETCD_HOSTS[@]}
do
# char '/' need to escape '\/'
initialCluster="${initialCluster}etcdnode${index}=http:\/\/${item}:2380"
if [[ ${index} -lt ${etcdHostsSize}-1 ]]; then
initialCluster=${initialCluster}","
fi
index=$(($index+1))
done
#echo "initialCluster=${initialCluster}"
sed -i "s/INITIAL_CLUSTER_REPLACE/${initialCluster}/g" $INSTALL_TEMP_DIR/etcd/etcd.service >>$LOG
cp $INSTALL_TEMP_DIR/etcd/etcd.service /etc/systemd/system/ >>$LOG
}
## @description install etcd
## @audience public
## @stability stable
function install_etcd()
{
index=$(indexByEtcdHosts ${LOCAL_HOST_IP})
if [ -z "$index" ]; then
echo -e "STOP: This host\033[31m[${LOCAL_HOST_IP}]\033[0m is not in the ETCD server list\033[31m[${ETCD_HOSTS[@]}]\033[0m"
return 1
fi
install_etcd_bin
install_etcd_config
systemctl daemon-reload
systemctl enable etcd.service
}
## @description uninstall etcd
## @audience public
## @stability stable
function uninstall_etcd()
{
echo "stop etcd.service"
systemctl stop etcd.service
echo "rm etcd ..."
rm /usr/bin/etcd
rm /usr/bin/etcdctl
rm -rf /var/lib/etcd
rm /etc/systemd/system/etcd.service
systemctl daemon-reload
}
## @description start etcd
## @audience public
## @stability stable
function start_etcd()
{
systemctl restart etcd.service
echo " ===== Check the status of the etcd service ====="
echo " exec etcdctl cluster-health"
etcdctl cluster-health
echo " exec etcdctl cluster-health"
etcdctl member list
}
## @description stop etcd
## @audience public
## @stability stable
function stop_etcd()
{
systemctl stop etcd.service
}

117
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/hadoop.sh
View File

@ -1,117 +0,0 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
## @description install yarn
## @audience public
## @stability stable
function install_yarn()
{
install_yarn_container_executor
install_yarn_config
}
## @description uninstall yarn
## @audience public
## @stability stable
function uninstall_yarn()
{
rm -rf /etc/yarn/sbin/Linux-amd64-64/*
rm -rf /etc/yarn/sbin/etc/hadoop/*
}
## @description download yarn container executor
## @audience public
## @stability stable
function download_yarn_container_executor()
{
# my download http server
if [[ -n "$DOWNLOAD_HTTP" ]]; then
MY_YARN_CONTAINER_EXECUTOR_PATH="${DOWNLOAD_HTTP}/downloads/hadoop/container-executor"
else
MY_YARN_CONTAINER_EXECUTOR_PATH=${YARN_CONTAINER_EXECUTOR_PATH}
fi
if [ ! -d "${DOWNLOAD_DIR}/hadoop" ]; then
mkdir -p ${DOWNLOAD_DIR}/hadoop
fi
if [[ -f "${DOWNLOAD_DIR}/hadoop/container-executor" ]]; then
echo "${DOWNLOAD_DIR}/hadoop/container-executor is exist."
else
if [[ -n "$DOWNLOAD_HTTP" ]]; then
echo "download ${MY_YARN_CONTAINER_EXECUTOR_PATH} ..."
wget -P ${DOWNLOAD_DIR}/hadoop ${MY_YARN_CONTAINER_EXECUTOR_PATH}
else
echo "copy ${MY_YARN_CONTAINER_EXECUTOR_PATH} ..."
cp ${MY_YARN_CONTAINER_EXECUTOR_PATH} ${DOWNLOAD_DIR}/hadoop/
fi
fi
}
## @description install yarn container executor
## @audience public
## @stability stable
function install_yarn_container_executor()
{
echo "install yarn container executor file ..."
download_yarn_container_executor
if [ ! -d "/etc/yarn/sbin/Linux-amd64-64" ]; then
mkdir -p /etc/yarn/sbin/Linux-amd64-64
fi
if [ -f "/etc/yarn/sbin/Linux-amd64-64/container-executor" ]; then
rm /etc/yarn/sbin/Linux-amd64-64/container-executor
fi
cp -f ${DOWNLOAD_DIR}/hadoop/container-executor /etc/yarn/sbin/Linux-amd64-64
sudo chmod 6755 /etc/yarn/sbin/Linux-amd64-64
sudo chown :yarn /etc/yarn/sbin/Linux-amd64-64/container-executor
sudo chmod 6050 /etc/yarn/sbin/Linux-amd64-64/container-executor
}
## @description install yarn config
## @audience public
## @stability stable
function install_yarn_config()
{
echo "install yarn config file ..."
cp -R ${PACKAGE_DIR}/hadoop ${INSTALL_TEMP_DIR}/
find="/"
replace="\/"
escape_yarn_nodemanager_local_dirs=${YARN_NODEMANAGER_LOCAL_DIRS//$find/$replace}
escape_yarn_nodemanager_log_dirs=${YARN_NODEMANAGER_LOG_DIRS//$find/$replace}
escape_yarn_hierarchy=${YARN_HIERARCHY//$find/$replace}
sed -i "s/YARN_NODEMANAGER_LOCAL_DIRS_REPLACE/${escape_yarn_nodemanager_local_dirs}/g" $INSTALL_TEMP_DIR/hadoop/container-executor.cfg >>$LOG
sed -i "s/YARN_NODEMANAGER_LOG_DIRS_REPLACE/${escape_yarn_nodemanager_log_dirs}/g" $INSTALL_TEMP_DIR/hadoop/container-executor.cfg >>$LOG
sed -i "s/DOCKER_REGISTRY_REPLACE/${DOCKER_REGISTRY}/g" $INSTALL_TEMP_DIR/hadoop/container-executor.cfg >>$LOG
sed -i "s/CALICO_NETWORK_NAME_REPLACE/${CALICO_NETWORK_NAME}/g" $INSTALL_TEMP_DIR/hadoop/container-executor.cfg >>$LOG
sed -i "s/YARN_HIERARCHY_REPLACE/${escape_yarn_hierarchy}/g" $INSTALL_TEMP_DIR/hadoop/container-executor.cfg >>$LOG
# Delete the ASF license comment in the container-executor.cfg file, otherwise it will cause a cfg format error.
sed -i '1,16d' $INSTALL_TEMP_DIR/hadoop/container-executor.cfg
if [ ! -d "/etc/yarn/sbin/etc/hadoop" ]; then
mkdir -p /etc/yarn/sbin/etc/hadoop
fi
cp -f $INSTALL_TEMP_DIR/hadoop/container-executor.cfg /etc/yarn/sbin/etc/hadoop/
}

444
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/menu.sh
View File

@ -1,444 +0,0 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
## @description main menu
## @audience public
## @stability stable
main_menu()
{
cat<<MENULIST
====================================================================================
SUBMARINE INSTALLER ${SUBMARINE_INSTALLER_VERSION}
HOST:${LOCAL_HOST_IP} DOWNLOAD_SERVER:http://${DOWNLOAD_SERVER_IP}:${DOWNLOAD_SERVER_PORT}
====================================================================================
[Main menu]
------------------------------------------------------------------------------------
MENULIST
echo -e " \e[32m1.prepare system environment [..]\e[0m"
echo -e " \e[32m2.install component [..]\e[0m"
echo -e " \e[32m3.uninstall component [..]\e[0m"
echo -e " \e[32m4.start component [..]\e[0m"
echo -e " \e[32m5.stop component [..]\e[0m"
echo -e " \e[32m6.start download server [..]\e[0m"
echo -e ""
echo -e " \e[32mq.quit\e[0m"
cat<<MENULIST
====================================================================================
MENULIST
echo -ne "Please input your choice [\e[32m1\e[0m-\e[32m6\e[0m,\e[32mq\e[0m(quit)]:"
}
## @description check menu
## @audience public
## @stability stable
check_menu()
{
cat<<MENULIST
====================================================================================
SUBMARINE INSTALLER ${SUBMARINE_INSTALLER_VERSION}
HOST:${LOCAL_HOST_IP} DOWNLOAD_SERVER:http://${DOWNLOAD_SERVER_IP}:${DOWNLOAD_SERVER_PORT}
====================================================================================
[Main menu] > [prepare system environment]
------------------------------------------------------------------------------------
MENULIST
echo -e " \e[32m1.prepare operation system\e[0m"
echo -e " \e[32m2.prepare operation system kernel\e[0m"
echo -e " \e[32m3.prepare GCC version\e[0m"
echo -e " \e[32m4.check GPU\e[0m"
echo -e " \e[32m5.prepare user&group\e[0m"
echo -e " \e[32m6.prepare nvidia environment\e[0m"
echo -e ""
echo -e " \e[32mb.back main menu\e[0m"
cat<<MENULIST
====================================================================================
MENULIST
echo -ne "Please input your choice [\e[32m1\e[0m-\e[32m5\e[0m,\e[32mb\e[0m(back)]:"
}
## @description install menu
## @audience public
## @stability stable
install_menu()
{
cat<<MENULIST
====================================================================================
SUBMARINE INSTALLER ${SUBMARINE_INSTALLER_VERSION}
HOST:${LOCAL_HOST_IP} DOWNLOAD_SERVER:http://${DOWNLOAD_SERVER_IP}:${DOWNLOAD_SERVER_PORT}
====================================================================================
[Main menu] > [install component]
------------------------------------------------------------------------------------
MENULIST
echo -e " \e[32m1.instll etcd\e[0m"
echo -e " \e[32m2.instll docker\e[0m"
echo -e " \e[32m3.instll calico network\e[0m"
echo -e " \e[32m4.instll nvidia driver\e[0m"
echo -e " \e[32m5.instll nvidia docker\e[0m"
echo -e " \e[32m6.instll yarn container-executor\e[0m"
echo -e " \e[32m7.instll submarine autorun script\e[0m"
echo -e ""
echo -e " \e[32mb.back main menu\e[0m"
cat<<MENULIST
====================================================================================
MENULIST
echo -ne "Please input your choice [\e[32m1\e[0m-\e[32m7\e[0m,\e[32mb\e[0m(back)]:"
}
## @description unstall menu
## @audience public
## @stability stable
uninstall_menu()
{
cat<<MENULIST
====================================================================================
SUBMARINE INSTALLER ${SUBMARINE_INSTALLER_VERSION}
HOST:${LOCAL_HOST_IP} DOWNLOAD_SERVER:http://${DOWNLOAD_SERVER_IP}:${DOWNLOAD_SERVER_PORT}
====================================================================================
[Main menu] > [uninstll component]
------------------------------------------------------------------------------------
MENULIST
echo -e " \e[32m1.uninstll etcd\e[0m"
echo -e " \e[32m2.uninstll docker\e[0m"
echo -e " \e[32m3.uninstll calico network\e[0m"
echo -e " \e[32m4.uninstll nvidia driver\e[0m"
echo -e " \e[32m5.uninstll nvidia docker\e[0m"
echo -e " \e[32m6.uninstll yarn container-executor\e[0m"
echo -e " \e[32m7.uninstll submarine autorun script\e[0m"
echo -e ""
echo -e " \e[32mb.back main menu\e[0m"
cat<<MENULIST
====================================================================================
MENULIST
echo -ne "Please input your choice [\e[32m1\e[0m-\e[32m7\e[0m,\e[32mb\e[0m(back)]:"
}
## @description start menu
## @audience public
## @stability stable
start_menu()
{
cat<<MENULIST
====================================================================================
SUBMARINE INSTALLER ${SUBMARINE_INSTALLER_VERSION}
HOST:${LOCAL_HOST_IP} DOWNLOAD_SERVER:http://${DOWNLOAD_SERVER_IP}:${DOWNLOAD_SERVER_PORT}
====================================================================================
[Main menu] > [stop component]
------------------------------------------------------------------------------------
MENULIST
echo -e " \e[32m1.start etcd\e[0m"
echo -e " \e[32m2.start docker\e[0m"
echo -e " \e[32m3.start calico network\e[0m"
echo -e ""
echo -e " \e[32mb.back main menu\e[0m"
cat<<MENULIST
====================================================================================
MENULIST
echo -ne "Please input your choice [\e[32m1\e[0m-\e[32m3\e[0m,\e[32mb\e[0m(back)]:"
}
## @description stop menu
## @audience public
## @stability stable
stop_menu()
{
cat<<MENULIST
====================================================================================
SUBMARINE INSTALLER ${SUBMARINE_INSTALLER_VERSION}
HOST:${LOCAL_HOST_IP} DOWNLOAD_SERVER:http://${DOWNLOAD_SERVER_IP}:${DOWNLOAD_SERVER_PORT}
====================================================================================
[Main menu] > [stop component]
------------------------------------------------------------------------------------
MENULIST
echo -e " \e[32m1.stop etcd\e[0m"
echo -e " \e[32m2.stop docker\e[0m"
echo -e " \e[32m3.stop calico network\e[0m"
echo -e ""
echo -e " \e[32mb.back main menu\e[0m"
cat<<MENULIST
====================================================================================
MENULIST
echo -ne "Please input your choice [\e[32m1\e[0m-\e[32m3\e[0m,\e[32mb\e[0m(back)]:"
}
## @description menu operation
## @audience public
## @stability stable
menu_index="0"
menu()
{
clear
# echo "menu_index-menu_choice=$menu_index-$menu_choice"
case $menu_index in
"0")
main_menu
;;
"1")
check_menu
;;
"2")
install_menu
;;
"3")
uninstall_menu
;;
"4")
start_menu
;;
"5")
stop_menu
;;
"6")
start_download_server
;;
"q")
exit 1
;;
*)
echo "error input!"
menu_index="0"
menu_choice="0"
main_menu
;;
esac
read menu_choice
}
## @description menu process
## @audience public
## @stability stable
menu_process()
{
process=0
unset myselect
# echo "debug=$menu_index-$menu_choice"
case "$menu_index-$menu_choice" in
"1-b"|"2-b"|"3-b"|"4-b"|"5-b"|"6-b")
menu_index="0"
menu_choice="0"
;;
# check system environment
"1-1")
myselect="y"
check_operationSystem
;;
"1-2")
myselect="y"
check_operationSystemKernel
;;
"1-3")
myselect="y"
check_gccVersion
;;
"1-4")
myselect="y"
check_GPU
;;
"1-5")
myselect="y"
check_userGroup
;;
"1-6")
myselect="y"
prepare_nvidia_environment
;;
# install component
"2-1")
echo -n "Do you want to install etcd?[y|n]"
read myselect
if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
then
install_etcd
fi
;;
"2-2")
echo -n "Do you want to install docker?[y|n]"
read myselect
if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
then
install_docker
fi
;;
"2-3")
echo -n "Do you want to install calico network?[y|n]"
read myselect
if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
then
install_calico
fi
;;
"2-4")
echo -n "Do you want to install nvidia driver?[y|n]"
read myselect
if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
then
install_nvidia
fi
;;
"2-5")
echo -n "Do you want to install nvidia docker?[y|n]"
read myselect
if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
then
install_nvidia_docker
fi
;;
"2-6")
echo -n "Do you want to install yarn container-executor?[y|n]"
read myselect
if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
then
install_yarn
fi
;;
"2-7")
echo -n "Do you want to install submarine auto start script?[y|n]"
read myselect
if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
then
install_submarine
fi
;;
# uninstall component
"3-1")
echo -n "Do you want to uninstall etcd?[y|n]"
read myselect
if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
then
uninstall_etcd
fi
;;
"3-2")
echo -n "Do you want to uninstall docker?[y|n]"
read myselect
if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
then
uninstall_docker
fi
;;
"3-3")
echo -n "Do you want to uninstall calico network?[y|n]"
read myselect
if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
then
uninstall_calico
fi
;;
"3-4")
echo -n "Do you want to uninstall nvidia driver?[y|n]"
read myselect
if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
then
uninstall_nvidia
fi
;;
"3-5")
echo -n "Do you want to uninstall nvidia docker?[y|n]"
read myselect
if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
then
uninstall_nvidia_docker
fi
;;
"3-6")
echo -n "Do you want to uninstall yarn container-executor?[y|n]"
read myselect
if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
then
uninstall_yarn
fi
;;
"3-7")
echo -n "Do you want to uninstall submarine autostart script?[y|n]"
read myselect
if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
then
uninstall_submarine
fi
;;
# startup component
"4-1")
echo -n "Do you want to startup etcd?[y|n]"
read myselect
if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
then
start_etcd
fi
;;
"4-2")
echo -n "Do you want to startup docker?[y|n]"
read myselect
if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
then
start_docker
fi
;;
"4-3")
echo -n "Do you want to startup calico network?[y|n]"
read myselect
if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
then
start_calico
fi
;;
# stop component
"5-1")
echo -n "Do you want to stop etcd?[y|n]"
read myselect
if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
then
stop_etcd
fi
;;
"5-2")
echo -n "Do you want to stop docker?[y|n]"
read myselect
if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
then
stop_docker
fi
;;
"5-3")
echo -n "Do you want to stop calico network?[y|n]"
read myselect
if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
then
stop_calico
fi
;;
esac
if [[ "$myselect" = "y" || "$myselect" = "Y" ]]
then
process=1
fi
# echo "process=$process"
return $process
}

99
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia-docker.sh
View File

@ -1,99 +0,0 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
## @description download nvidia docker bin
## @audience public
## @stability stable
function download_nvidia_docker_bin()
{
# download http server
if [[ -n "$DOWNLOAD_HTTP" ]]; then
MY_NVIDIA_DOCKER_RPM_URL="${DOWNLOAD_HTTP}/downloads/nvidia-docker/${NVIDIA_DOCKER_RPM}"
else
MY_NVIDIA_DOCKER_RPM_URL=${NVIDIA_DOCKER_RPM_URL}
fi
if [[ -f "${DOWNLOAD_DIR}/nvidia-docker/${NVIDIA_DOCKER_RPM}" ]]; then
echo "${DOWNLOAD_DIR}/nvidia-docker/${NVIDIA_DOCKER_RPM} is exist."
else
echo "download ${MY_NVIDIA_DOCKER_RPM_URL} ..."
wget -P ${DOWNLOAD_DIR}/nvidia-docker/ ${MY_NVIDIA_DOCKER_RPM_URL}
fi
}
## @description install nvidia docker
## @audience public
## @stability stable
function install_nvidia_docker()
{
download_nvidia_docker_bin
sudo rpm -i ${DOWNLOAD_DIR}/nvidia-docker/${NVIDIA_DOCKER_RPM}
echo -e "\033[32m===== Start nvidia-docker =====\033[0m"
sudo systemctl start nvidia-docker
echo -e "\033[32m===== Check nvidia-docker status =====\033[0m"
systemctl status nvidia-docker
echo -e "\033[32m===== Check nvidia-docker log =====\033[0m"
journalctl -u nvidia-docker
echo -e "\033[32m===== Test nvidia-docker-plugin =====\033[0m"
curl http://localhost:3476/v1.0/docker/cli
# create nvidia driver library path
if [ ! -d "/var/lib/nvidia-docker/volumes/nvidia_driver" ]; then
echo "WARN: /var/lib/nvidia-docker/volumes/nvidia_driver folder path is not exist!"
mkdir -p /var/lib/nvidia-docker/volumes/nvidia_driver
fi
local nvidiaVersion=`get_nvidia_version`
echo -e "\033[31m nvidia detect version is ${nvidiaVersion}\033[0m"
mkdir /var/lib/nvidia-docker/volumes/nvidia_driver/${nvidiaVersion}
mkdir /var/lib/nvidia-docker/volumes/nvidia_driver/${nvidiaVersion}/bin
mkdir /var/lib/nvidia-docker/volumes/nvidia_driver/${nvidiaVersion}/lib64
cp /usr/bin/nvidia* /var/lib/nvidia-docker/volumes/nvidia_driver/${nvidiaVersion}/bin
cp /usr/lib64/libcuda* /var/lib/nvidia-docker/volumes/nvidia_driver/${nvidiaVersion}/lib64
cp /usr/lib64/libnvidia* /var/lib/nvidia-docker/volumes/nvidia_driver/${nvidiaVersion}/lib64
echo -e "\033[32m===== Please manually execute the following command =====\033[0m"
echo -e "\033[32mshell:> nvidia-docker run --rm ${DOCKER_REGISTRY}/nvidia/cuda:9.0-devel nvidia-smi
# If you don't see the list of graphics cards above, the NVIDIA driver installation failed. =====
\033[0m"
echo -e "\033[32m===== Please manually execute the following command =====\033[0m"
echo -e "\033[32m# Test with tf.test.is_gpu_available()
shell:> nvidia-docker run -it ${DOCKER_REGISTRY}/tensorflow/tensorflow:1.9.0-gpu bash
# In docker container
container:> python
python:> import tensorflow as tf
python:> tf.test.is_gpu_available()
python:> exit()
\033[0m"
}
## @description uninstall nvidia docker
## @audience public
## @stability stable
function uninstall_nvidia_docker()
{
echo "This method is not implemented."
}

120
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia.sh
View File

@ -1,120 +0,0 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
nvidia_run_file=""
## @description get nvidia version
## @audience public
## @stability stable
function get_nvidia_version()
{
chmod +x ${DOWNLOAD_DIR}/nvidia/nvidia-detect
local nvidia_detect_info=`${DOWNLOAD_DIR}/nvidia/nvidia-detect -v`
echo $nvidia_detect_info | sed "s/^.*This device requires the current \([0-9.]*\).*/\1/"
}
## @description download nvidia driver
## @audience public
## @stability stable
function download_nvidia_driver()
{
# submarin http server
if [[ -n "$DOWNLOAD_HTTP" ]]; then
MY_NVIDIA_DETECT_URL="${DOWNLOAD_HTTP}/downloads/nvidia/nvidia-detect"
else
MY_NVIDIA_DETECT_URL=${NVIDIA_DETECT_URL}
fi
# download nvidia_detect
if [[ -f "${DOWNLOAD_DIR}/nvidia/nvidia-detect" ]]; then
echo "${DOWNLOAD_DIR}/nvidia/nvidia-detect is exist."
else
echo "download ${MY_NVIDIA_DETECT_URL} ..."
wget -P ${DOWNLOAD_DIR}/nvidia/ ${MY_NVIDIA_DETECT_URL}
fi
echo "execution nvidia-detect to check the graphics card ..."
local nvidiaVersion=`get_nvidia_version`
echo -e "detect nvidia version is \033[31m${nvidiaVersion}\033[0m"
# download NVIDIA driver
if [[ "$nvidiaVersion" = "" ]]; then
echo -e "\033[31mERROR: No graphics card device detected.\033[0m"
return 1
else
nvidia_run_file="NVIDIA-Linux-x86_64-${nvidiaVersion}.run"
# submarin http server
if [[ -n "$DOWNLOAD_HTTP" ]]; then
MY_NVIDIA_DRIVER_RUN_URL="${DOWNLOAD_HTTP}/downloads/nvidia/${nvidia_run_file}"
else
# http://us.download.nvidia.com/XFree86/Linux-x86_64/390.87/NVIDIA-Linux-x86_64-390.87.run
MY_NVIDIA_DRIVER_RUN_URL="http://us.download.nvidia.com/XFree86/Linux-x86_64/${nvidiaVersion}/${nvidia_run_file}"
fi
if [[ -f ${DOWNLOAD_DIR}/nvidia/${nvidia_run_file} ]]; then
echo "NVIDIA driver files already exist in the ${DOWNLOAD_DIR}/nvidia/${nvidia_run_file} directory."
echo "===== Please make sure the ${DOWNLOAD_DIR}/nvidia/nvidia/${nvidia_run_file} file is complete and can be used normally. ====="
else
echo "Download the NVIDIA driver from the ${MY_NVIDIA_DRIVER_RUN_URL}"
wget -P ${DOWNLOAD_DIR}/nvidia/ ${MY_NVIDIA_DRIVER_RUN_URL}
fi
fi
}
## @description install nvidia
## @audience public
## @stability stable
function install_nvidia()
{
download_nvidia_driver
# Confirm that the system disables nouveau
local disable_nouveau_info=`lsmod | grep nouveau`
if [[ "$disable_nouveau_info" = "" ]]; then
echo "===== Start installing the NVIDIA driver ====="
echo -e "Some options during the installation
Would you like to register the kernel module sources with DKMS?
This will allow DKMS to automatically build a new module, if you install a different kernel later. \033[33m[Yes]\033[0m
Install NVIDIA's 32-bit compatibility libraries \033[33m[Yes]\033[0m
centos Install NVIDIA's 32-bit compatibility libraries \033[33m[Yes]\033[0m
Would you like to run the nvidia-xconfig utility to automatically update your X configuration file... \033[33m[No]\033[0m"
sleep 2
sh ${DOWNLOAD_DIR}/nvidia/${nvidia_run_file}
else
echo -e "ERROR: Nouveau is not disabled"
return 1
fi
echo -e "\033[32m===== execute nvidia-smi. You should be able to see the list of graphics cards =====\033[0m"
sleep 1
nvidia-smi
}
## @description uninstall nvidia
## @audience public
## @stability stable
function uninstall_nvidia()
{
if [ ! -f "/usr/bin/nvidia-uninstall" ]; then
echo -e "\033[31mERROR: /usr/bin/nvidia-uninstall file is not exist!\033[0m"
return 1
fi
echo -e "execute /usr/bin/nvidia-uninstall"
/usr/bin/nvidia-uninstall
}

38
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/submarine.sh
View File

@ -1,38 +0,0 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
## @description install submarine autorun script
## @audience public
## @stability stable
function install_submarine()
{
cp ${PACKAGE_DIR}/submarine/submarine.sh /etc/rc.d/init.d/submarine.sh
chmod +x /etc/rc.d/init.d/submarine.sh
chmod +x /etc/rc.d/rc.local
if [ `grep -c "/etc/rc.d/init.d/submarine.sh" /etc/rc.d/rc.local` -eq '0' ]; then
echo "/etc/rc.d/init.d/submarine.sh">> /etc/rc.d/rc.local
fi
}
## @description uninstall submarine autorun script
## @audience public
## @stability stable
function uninstall_submarine()
{
rm /etc/rc.d/init.d/submarine.sh
}

123
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh
View File

@ -1,123 +0,0 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
## @description check install user
## @audience public
## @stability stable
function check_install_user()
{
if [[ $(id -u) -ne 0 ]];then
echo "This script must be run with a ROOT user!"
exit # don't call exit_install()
fi
}
## @description exit install
## @audience public
## @stability stable
function exit_install()
{
echo "Exit the installation!" | tee -a $LOG
exit $1
}
## @description Check if the IP address format is correct
## @audience public
## @stability stable
function valid_ip()
{
local ip=$1
local stat=1
if [[ $ip =~ ^[0-9]{1,3\}.[0-9]{1,3\}.[0-9]{1,3\}.[0-9]{1,3\}$ ]]; then
OIFS=$IFS
IFS='.'
ip=($ip)
IFS=$OIFS
if [[ ${ip[0]} -le 255 && ${ip[1]} -le 255 && ${ip[2]} -le 255 && ${ip[3]} -le 255 ]]; then
stat=$?
fi
fi
return $stat
}
## @description Check if the configuration file configuration is correct
## @audience public
## @stability stable
function check_install_conf()
{
echo "Check if the configuration file configuration is correct ..." | tee -a $LOG
# check etcd conf
hostCount=${#ETCD_HOSTS[@]}
if [[ $hostCount -lt 3 && hostCount -ne 0 ]]; then # <>2
echo "Number of nodes = [$hostCount], must be configured to be greater than or equal to 3 servers! " | tee -a $LOG
exit_install
fi
for ip in ${ETCD_HOSTS[@]}
do
if ! valid_ip $ip; then
echo "]ETCD_HOSTS=[$ip], IP address format is incorrect! " | tee -a $LOG
exit_install
fi
done
echo "Check if the configuration file configuration is correct [ Done ]" | tee -a $LOG
}
## @description index by EtcdHosts list
## @audience public
## @stability stable
function indexByEtcdHosts() {
index=0
while [ "$index" -lt "${#ETCD_HOSTS[@]}" ]; do
if [ "${ETCD_HOSTS[$index]}" = "$1" ]; then
echo $index
return
fi
let "index++"
done
echo ""
}
## @description get local IP
## @audience public
## @stability stable
function getLocalIP()
{
local _ip _myip _line _nl=$'\n'
while IFS=$': \t' read -a _line ;do
[ -z "${_line%inet}" ] &&
_ip=${_line[${#_line[1]}>4?1:2]} &&
[ "${_ip#127.0.0.1}" ] && _myip=$_ip
done< <(LANG=C /sbin/ifconfig)
printf ${1+-v} $1 "%s${_nl:0:$[${#1}>0?0:1]}" $_myip
}
## @description get ip list
## @audience public
## @stability stable
function get_ip_list()
{
array=$(ifconfig | grep inet | grep -v inet6 | grep -v 127 | sed 's/^[ \t]*//g' | cut -d ' ' -f2)
for ip in ${array[@]}
do
LOCAL_HOST_IP_LIST+=(${ip})
done
}