diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml index a2ea08c602..9c401e8a7c 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml @@ -65,6 +65,14 @@ 0755 + + hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation + /share/hadoop/yarn/submarine-installer + + **/* + + 0755 + hadoop-yarn/conf etc/hadoop diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf new file mode 100644 index 0000000000..82dcf61b73 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +##### install config ##### + +# DNS +LOCAL_DNS_HOST="172.17.0.9" # /etc/resolv.conf +YARN_DNS_HOST="10.196.69.173" # yarn dns server ip address + +# etcd hosts list +ETCD_HOSTS=(10.196.69.173 10.196.69.174 10.196.69.175) + +# docker registry ip:port +DOCKER_REGISTRY="10.120.196.232:5000" + +# Start the http download service on the specified server, +# Will download all the dependencies in the http server, +# Run the install script on other servers. +# Automatically download dependencies from http, +# Solve the problem that all servers are slow to download online. +# At the same time, you can also manually download the dependencies to the downloads directory. +# Offline installation of the system +DOWNLOAD_SERVER_IP="10.120.196.236" +DOWNLOAD_SERVER_PORT="19000" + +# yarn container-executor config +# How to compile container-executor: +# Go to the hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager +# path and enter the command: mvn package -Pnative -DskipTests +# Only the nodemanager is compiled. The path of the compiled container-executor is: +# ./target/native/target/usr/local/bin/container-executor +YARN_CONTAINER_EXECUTOR_PATH="Please enter the full path to the container-executor here" + +# Keep the same configuration as 'yarn.nodemanager.linux-container-executor.cgroups.hierarchy' +# in yarn-site.xml, default '/hadoop-yarn' +YARN_HIERARCHY="/hadoop-yarn" + +# Keep the same configuration as 'yarn.nodemanager.local-dirs' in yarn-site.xml +YARN_NODEMANAGER_LOCAL_DIRS="/home/hadoop/disk/1/yarn/local,/home/hadoop/disk/2/yarn/local,/home/hadoop/disk/3/yarn/local" + +# Keep the same configuration as 'yarn.nodemanager.log-dirs' in yarn-site.xml +YARN_NODEMANAGER_LOG_DIRS="/home/hadoop/disk/1/yarn/logs,/home/hadoop/disk/2/yarn/logs,/home/hadoop/disk/3/yarn/logs" + +##### System component download url address, Generally do not need to be modified ##### +DOCKER_REPO="https://yum.dockerproject.org/repo/main/centos/7/Packages" +DOCKER_ENGINE_RPM="docker-engine-1.12.5-1.el7.centos.x86_64.rpm" +DOCKER_ENGINE_SELINUX_RPM="docker-engine-selinux-1.12.5-1.el7.centos.noarch.rpm" + +NVIDIA_DETECT_URL="https://raw.githubusercontent.com/liuxunorg/submarine-installer/packages/nvidia/nvidia-detect" +NVIDIA_DOCKER_RPM="nvidia-docker-1.0.1-1.x86_64.rpm" +NVIDIA_DOCKER_RPM_URL="https://github.com/NVIDIA/nvidia-docker/releases/download/v1.0.1/${NVIDIA_DOCKER_RPM}" + +ETCD_TAR_GZ="etcd-v3.3.9-linux-amd64.tar.gz" +ETCD_DOWNLOAD_URL="https://github.com/etcd-io/etcd/releases/download/v3.3.9/${ETCD_TAR_GZ}" + +CALICO_IPV4POOL_CIDR="192.20.0.0" +CALICO_NETWORK_NAME="calico-network" +CALICO_DOWNLOAD_URL="https://github.com/projectcalico/cni-plugin/releases/download/v1.11.7/calico" +CALICO_IPAM_DOWNLOAD_URL="https://github.com/projectcalico/cni-plugin/releases/download/v1.11.7/calico-ipam" +CALICOCTL_DOWNLOAD_URL="https://github.com/projectcalico/calicoctl/releases/download/v3.2.3/calicoctl" diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.sh new file mode 100755 index 0000000000..90b12c2820 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# description: sumbarine install scripts. + +ROOT=$(cd "$(dirname "$0")"; pwd) +SUBMARINE_INSTALLER_VERSION="v0.7" +PACKAGE_DIR=${ROOT}/package +SCRIPTS_DIR=${ROOT}/scripts +INSTALL_TEMP_DIR=${ROOT}/temp +DOWNLOAD_DIR=${ROOT}/downloads +DATE=`date +%Y%m%d-%H:%M:%S` +LOG=${ROOT}/logs/install.log.`date +%Y%m%d%H%M%S` +LOCAL_HOST_IP_LIST=() +LOCAL_HOST_IP="" +OPERATING_SYSTEM="" +DOWNLOAD_HTTP="" + +# import shell script +. ${ROOT}/install.conf +. ${ROOT}/scripts/calico.sh +. ${ROOT}/scripts/docker.sh +. ${ROOT}/scripts/download-server.sh +. ${ROOT}/scripts/environment.sh +. ${ROOT}/scripts/etcd.sh +. ${ROOT}/scripts/hadoop.sh +. ${ROOT}/scripts/menu.sh +. ${ROOT}/scripts/nvidia.sh +. ${ROOT}/scripts/nvidia-docker.sh +. ${ROOT}/scripts/submarine.sh +. ${ROOT}/scripts/utils.sh + +#================================= Main ======================================== +mkdir $ROOT/logs/ -p +mkdir $INSTALL_TEMP_DIR -p +mkdir $DOWNLOAD_DIR -p + +source /etc/os-release +OPERATING_SYSTEM=$ID + +get_ip_list +ipCount=${#LOCAL_HOST_IP_LIST[@]} +if [[ $ipCount -eq 1 ]]; then + LOCAL_HOST_IP=${LOCAL_HOST_IP_LIST[0]} +else + echo -e "Detect the network card IP in the server, \e[31m[${LOCAL_HOST_IP_LIST[@]}]\e[0m" + echo -n -e "please enter a valid IP address: " + + read ipInput + if ! valid_ip $ipInput; then + echo -e "you input \e[31m$ipInput\e[0m address format is incorrect! " | tee -a $LOG + exit_install + else + LOCAL_HOST_IP=$ipInput + fi +fi + +echo -n -e "Please confirm whether the IP address of this machine is \e[31m${LOCAL_HOST_IP}\e[0m?[y|n]" +read myselect +if [[ "$myselect" != "y" && "$myselect" != "Y" ]]; then + exit_install +fi + +check_install_conf + +if [[ -n "$DOWNLOAD_SERVER_IP" && -n "$DOWNLOAD_SERVER_PORT" && "$DOWNLOAD_SERVER_IP" != "$LOCAL_HOST_IP" ]]; then + DOWNLOAD_HTTP="http://${DOWNLOAD_SERVER_IP}:${DOWNLOAD_SERVER_PORT}" +fi + +check_install_user + +# 清理安装临时目录 +rm $INSTALL_TEMP_DIR/* -rf >>$LOG 2>&1 + +menu_index="0" +for ((j=1;;j++)) +do + menu + case "$menu_index" in + "0") + menu_index="$menu_choice" + ;; + "1"|"2"|"3"|"4"|"5") +# echo "aaaa=$menu_index-$menu_choice" + menu_process + if [[ $? = 1 ]]; then + echo "Press any key to return menu!" + read + fi + ;; + "a") + exit_install + ;; + "q") + exit_install + ;; + *) + menu_index="0" + menu_choice="0" + menu + ;; + esac +done diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calico-node.service b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calico-node.service new file mode 100644 index 0000000000..744c4191f9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calico-node.service @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[Unit] +Description=calico node +After=docker.service +Requires=docker.service + +[Service] +User=root +PermissionsStartOnly=true +ExecStart=/usr/bin/docker run --net=host --privileged --name=calico-node \ + -e ETCD_ENDPOINTS=ETCD_ENDPOINTS_REPLACE \ + -e CALICO_LIBNETWORK_ENABLED=true \ + -e CALICO_NETWORKING_BACKEND=bird \ + -e CALICO_DISABLE_FILE_LOGGING=true \ + -e CALICO_IPV4POOL_CIDR=CALICO_IPV4POOL_CIDR_REPLACE/16 \ + -e CALICO_IPV4POOL_IPIP=always \ + -e FELIX_DEFAULTENDPOINTTOHOSTACTION=ACCEPT \ + -e FELIX_IPV6SUPPORT=false \ + -e FELIX_LOGSEVERITYSCREEN=info \ + -e FELIX_IPINIPMTU=1440 \ + -e FELIX_HEALTHENABLED=true \ + -e IP= \ + -v /etc/calico/ssl:/etc/calico/ssl \ + -v /var/run/calico:/var/run/calico \ + -v /lib/modules:/lib/modules \ + -v /run/docker/plugins:/run/docker/plugins \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /var/log/calico:/var/log/calico \ + calico/node:v2.6.2 +ExecStop=/usr/bin/docker rm -f calico-node +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calicoctl.cfg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calicoctl.cfg new file mode 100644 index 0000000000..44ada867a6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calicoctl.cfg @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: calicoApiConfig +metadata: +spec: + datastoreType: "etcdv2" + etcdEndpoints: ETCD_ENDPOINTS_REPLACE \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/daemon.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/daemon.json new file mode 100644 index 0000000000..052fe732b8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/daemon.json @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{ + "insecure-registries": ["DOCKER_REGISTRY_REPLACE"], + "cluster-store":"CLUSTER_STORE_REPLACE", + "cluster-advertise":"LOCAL_HOST_IP_REPLACE:2375", + "dns": ["YARN_DNS_HOST_REPLACE", "LOCAL_DNS_HOST_REPLACE"], + "hosts": ["tcp://LOCAL_HOST_IP_REPLACE:2375","unix:///var/run/docker.sock"] +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/docker.service b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/docker.service new file mode 100644 index 0000000000..f444f995d5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/docker.service @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[Unit] +Description=Docker Application Container Engine +Documentation=http://docs.docker.io + +[Service] +Environment="PATH=/usr/bin:/bin:/sbin:/usr/bin:/usr/sbin" +ExecStart=/usr/bin/dockerd --log-level=error +ExecStartPost=/sbin/iptables -I FORWARD -s 0.0.0.0/0 -j ACCEPT +ExecReload=/bin/kill -s HUP $MAINPID +Restart=on-failure +RestartSec=5 +LimitNOFILE=infinity +LimitNPROC=infinity +LimitCORE=infinity +Delegate=yes +KillMode=process + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/etcd/etcd.service b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/etcd/etcd.service new file mode 100644 index 0000000000..5a44ce2cfa --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/etcd/etcd.service @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[Unit] +Description=etcd +Documentation=https://github.com/coreos/etcd + +[Service] +Type=notify +Restart=always +RestartSec=5s +LimitNOFILE=40000 +TimeoutStartSec=0 + +ExecStart=/usr/bin/etcd \ + --name=ETCD_NODE_NAME_REPLACE \ + --data-dir=/var/lib/etcd \ + --listen-client-urls=http://LOCAL_HOST_REPLACE:2379,http://127.0.0.1:2379 \ + --listen-peer-urls=http://LOCAL_HOST_REPLACE:2380 \ + --advertise-client-urls=http://LOCAL_HOST_REPLACE:2379 \ + --initial-advertise-peer-urls=http://LOCAL_HOST_REPLACE:2380 \ + --initial-cluster=INITIAL_CLUSTER_REPLACE \ + --initial-cluster-token=etcd-token \ + --initial-cluster-state=new + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/hadoop/container-executor.cfg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/hadoop/container-executor.cfg new file mode 100644 index 0000000000..9d9f0cade0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/hadoop/container-executor.cfg @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +yarn.nodemanager.local-dirs=YARN_NODEMANAGER_LOCAL_DIRS_REPLACE +yarn.nodemanager.linux-container-executor.group=yarn +yarn.nodemanager.log-dirs=YARN_NODEMANAGER_LOG_DIRS_REPLACE +banned.users=root +allowed.system.users=yarn +min.user.id=500 + +[docker] +module.enabled=true +docker.binary=/usr/bin/docker +docker.allowed.capabilities=SYS_CHROOT,MKNOD,SETFCAP,SETPCAP,FSETID,CHOWN,AUDIT_WRITE,SETGID,NET_RAW,FOWNER,SETUID,DAC_OVERRIDE,KILL,NET_BIND_SERVICE,DAC_READ_SEARCH,SYS_PTRACE,SYS_ADMIN +docker.allowed.networks=bridge,host,none,CALICO_NETWORK_NAME_REPLACE +docker.allowed.ro-mounts=/etc/group,/etc/passwd,/etc/krb5.conf,YARN_NODEMANAGER_LOCAL_DIRS_REPLACE,regex:^nvidia_driver_.*$ +docker.allowed.rw-mounts=YARN_NODEMANAGER_LOCAL_DIRS_REPLACE,YARN_NODEMANAGER_LOG_DIRS_REPLACE +docker.privileged-containers.enabled=false +docker.trusted.registries=local,centos,hortonworks,DOCKER_REGISTRY_REPLACE +docker.allowed.volume-drivers=nvidia-docker +docker.allowed.devices=regex:^/dev/nvidia.*$ + +[gpu] +module.enabled=true + +[cgroups] +root=/sys/fs/cgroup +yarn-hierarchy=YARN_HIERARCHY_REPLACE \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/submarine/submarine.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/submarine/submarine.sh new file mode 100644 index 0000000000..ffbc38f75a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/submarine/submarine.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Solution: Grant user yarn the access to /sys/fs/cgroup/cpu,cpuacct, +# which is the subfolder of cgroup mount destination. +chown :yarn -R /sys/fs/cgroup/cpu,cpuacct +chmod g+rwx -R /sys/fs/cgroup/cpu,cpuacct + +# If GPUs are used，the access to cgroup devices folder is neede as well +chown :yarn -R /sys/fs/cgroup/devices +chmod g+rwx -R /sys/fs/cgroup/devices diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/calico.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/calico.sh new file mode 100644 index 0000000000..bd40fcfe3a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/calico.sh @@ -0,0 +1,224 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description download calico bin +## @audience public +## @stability stable +function download_calico_bin() +{ + # submarin http server + if [[ -n "$DOWNLOAD_HTTP" ]]; then + MY_CALICOCTL_DOWNLOAD_URL=${DOWNLOAD_HTTP}/downloads/calico/calicoctl + MY_CALICO_DOWNLOAD_URL=${DOWNLOAD_HTTP}/downloads/calico/calico + MY_CALICO_IPAM_DOWNLOAD_URL=${DOWNLOAD_HTTP}/downloads/calico/calico-ipam + else + MY_CALICOCTL_DOWNLOAD_URL=${CALICOCTL_DOWNLOAD_URL} + MY_CALICO_DOWNLOAD_URL=${CALICO_DOWNLOAD_URL} + MY_CALICO_IPAM_DOWNLOAD_URL=${CALICO_IPAM_DOWNLOAD_URL} + fi + + mkdir -p ${DOWNLOAD_DIR}/calico + + if [[ -f ${DOWNLOAD_DIR}/calico/calico ]]; then + echo "${DOWNLOAD_DIR}/calico/calico is exist." + else + echo "download ${MY_CALICO_DOWNLOAD_URL} ..." + wget -P ${DOWNLOAD_DIR}/calico ${MY_CALICO_DOWNLOAD_URL} + fi + + if [[ -f ${DOWNLOAD_DIR}/calico/calicoctl ]]; then + echo "${DOWNLOAD_DIR}/calico is exist." + else + echo "download ${MY_CALICOCTL_DOWNLOAD_URL} ..." + wget -P ${DOWNLOAD_DIR}/calico ${MY_CALICOCTL_DOWNLOAD_URL} + fi + + if [[ -f ${DOWNLOAD_DIR}/calico/calico-ipam ]]; then + echo "${DOWNLOAD_DIR}/calico/calico-ipam is exist." + else + echo "download ${MY_CALICO_IPAM_DOWNLOAD_URL} ..." + wget -P ${DOWNLOAD_DIR}/calico ${MY_CALICO_IPAM_DOWNLOAD_URL} + fi +} + +## @description install calico bin +## @audience public +## @stability stable +function install_calico_bin() +{ + download_calico_bin + + cp -f ${DOWNLOAD_DIR}/calico/calico /usr/bin/calico + cp -f ${DOWNLOAD_DIR}/calico/calicoctl /usr/bin/calicoctl + cp -f ${DOWNLOAD_DIR}/calico/calico-ipam /usr/bin/calico-ipam + + chmod +x /usr/bin/calico + chmod +x /usr/bin/calicoctl + chmod +x /usr/bin/calico-ipam +} + +## @description install calico config +## @audience public +## @stability stable +function install_calico_config() +{ + mkdir -p /etc/calico + + cp -rf ${PACKAGE_DIR}/calico ${INSTALL_TEMP_DIR}/ + + # 1. replace etcdEndpoints + # etcdEndpoints: https://10.196.69.173:2379,https://10.196.69.174:2379,https://10.196.69.175:2379 + etcdEndpoints='' + index=0 + etcdHostsSize=${#ETCD_HOSTS[@]} + for item in ${ETCD_HOSTS[@]} + do + index=$(($index+1)) + etcdEndpoints="${etcdEndpoints}http:\/\/${item}:2379" + if [[ ${index} -lt ${etcdHostsSize} ]]; then + etcdEndpoints=${etcdEndpoints}"," + fi + done + # echo "etcdEndpoints=${etcdEndpoints}" + sed -i "s/ETCD_ENDPOINTS_REPLACE/${etcdEndpoints}/g" $INSTALL_TEMP_DIR/calico/calicoctl.cfg >>$LOG + + if [[ ! -d /etc/calico ]]; then + mkdir /etc/calico + else + rm -rf /etc/calico/* + fi + + cp -f $INSTALL_TEMP_DIR/calico/calicoctl.cfg /etc/calico/calicoctl.cfg + + sed -i "s/ETCD_ENDPOINTS_REPLACE/${etcdEndpoints}/g" $INSTALL_TEMP_DIR/calico/calico-node.service >>$LOG + sed -i "s/CALICO_IPV4POOL_CIDR_REPLACE/${CALICO_IPV4POOL_CIDR}/g" $INSTALL_TEMP_DIR/calico/calico-node.service >>$LOG + cp $INSTALL_TEMP_DIR/calico/calico-node.service /etc/systemd/system/ >>$LOG + + systemctl daemon-reload + systemctl enable calico-node.service +} + +## @description modify kernel network config +## @audience public +## @stability stable +function kernel_network_config() +{ + if [ `grep -c "net.ipv4.conf.all.rp_filter=1" /etc/sysctl.conf` -eq '0' ]; then + echo "net.ipv4.conf.all.rp_filter=1" >>/etc/sysctl.conf + fi + + if [ `grep -c "net.ipv4.ip_forward=1" /etc/sysctl.conf` -eq '0' ]; then + echo "net.ipv4.ip_forward=1" >>/etc/sysctl.conf + fi + + sysctl -p +} + +## @description check if the calico-network exist +## @audience public +## @stability stable +function calico_network_exist() +{ + local dockerNetwokInfo=`docker network ls --filter NAME=${CALICO_NETWORK_NAME}` + echo ${dockerNetwokInfo} | grep ${CALICO_NETWORK_NAME} +} + +## @description verification calico +## @audience public +## @stability stable +function verification_calico() +{ + echo " ===== Check if the network between 2 containers can be connected =====" + local claicoNetworkExist=`calico_network_exist` + if [[ "$claicoNetworkExist" = "" ]]; then + echo "Create a calico network" + docker network create --driver calico --ipam-driver calico-ipam ${CALICO_NETWORK_NAME} + else + echo "calico network ${CALICO_NETWORK_NAME} is exist." + fi + + local verifyA="verify-calico-network-A" + local verifyAInfo=`containers_exist ${verifyA}` + if [[ -n "$verifyAInfo" ]]; then + echo "Delete existing container ${verifyA}." + docker stop ${verifyA} + docker rm ${verifyA} + fi + echo "Create containers verify-calico-network-A" + docker run --net ${CALICO_NETWORK_NAME} --name ${verifyA} -tid busybox + + local verifyB="verify-calico-network-B" + local verifyBInfo=`containers_exist ${verifyB}` + if [[ -n "$verifyBInfo" ]]; then + echo "Delete existing container ${verifyB}." + docker stop ${verifyB} + docker rm ${verifyB} + fi + echo "Create containers verify-calico-network-B" + docker run --net ${CALICO_NETWORK_NAME} --name ${verifyB} -tid busybox + + echo -e "\033[33m${verifyA} ping ${verifyB}\033[0m" + docker exec ${verifyA} ping ${verifyB} -c 5 +} + +## @description install calico +## @audience public +## @stability stable +function install_calico() +{ + kernel_network_config + install_calico_bin + install_calico_config + start_calico + verification_calico +} + +## @description uninstall calico +## @audience public +## @stability stable +function uninstall_calico() +{ + echo "stop calico-node.service" + systemctl stop calico-node.service + + echo "rm /usr/bin/calico ..." + rm /usr/bin/calicoctl + rm /usr/bin/calico + rm /usr/bin/calico-ipam + + rm -rf /etc/calico/ + rm /etc/systemd/system/calico-node.service + systemctl daemon-reload +} + +## @description start calico +## @audience public +## @stability stable +function start_calico() +{ + systemctl restart calico-node.service + systemctl status calico-node.service +} + +## @description stop calico +## @audience public +## @stability stable +function stop_calico() +{ + systemctl stop calico-node.service + systemctl status calico-node.service +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/docker.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/docker.sh new file mode 100644 index 0000000000..26f19cc9a9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/docker.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description download docker rmp +## @audience public +## @stability stable +function download_docker_rpm() +{ + # download http server + if [[ -n "$DOWNLOAD_HTTP" ]]; then + MY_DOCKER_ENGINE_SELINUX_RPM="${DOWNLOAD_HTTP}/downloads/docker/${DOCKER_ENGINE_SELINUX_RPM}" + MY_DOCKER_ENGINE_RPM="${DOWNLOAD_HTTP}/downloads/docker/${DOCKER_ENGINE_RPM}" + else + MY_DOCKER_ENGINE_SELINUX_RPM=${DOCKER_REPO}/${DOCKER_ENGINE_SELINUX_RPM} + MY_DOCKER_ENGINE_RPM=${DOCKER_REPO}/${DOCKER_ENGINE_RPM} + fi + + # download docker rpm + if [[ -f ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_SELINUX_RPM} ]]; then + echo "${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_SELINUX_RPM} is exist." + else + echo "download ${MY_DOCKER_ENGINE_SELINUX_RPM} ..." + wget -P ${DOWNLOAD_DIR}/docker/ ${MY_DOCKER_ENGINE_SELINUX_RPM} + fi + + if [[ -f ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_RPM} ]]; then + echo "${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_RPM} is exist." + else + echo "download ${MY_DOCKER_ENGINE_RPM} ..." + wget -P ${DOWNLOAD_DIR}/docker/ ${MY_DOCKER_ENGINE_RPM} + fi +} + +## @description install docker bin +## @audience public +## @stability stable +function install_docker_bin() +{ + download_docker_rpm + + yum -y localinstall ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_SELINUX_RPM} + yum -y localinstall ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_RPM} +} + +## @description uninstall docker bin +## @audience public +## @stability stable +function uninstall_docker_bin() +{ + download_docker_rpm + + yum -y remove ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_SELINUX_RPM} + yum -y remove ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_RPM} +} + +## @description install docker config +## @audience public +## @stability stable +function install_docker_config() +{ + rm -rf ${INSTALL_TEMP_DIR}/docker + cp -rf ${PACKAGE_DIR}/docker ${INSTALL_TEMP_DIR}/ + + # replace cluster-store + # "cluster-store":"etcd://10.196.69.173:2379,10.196.69.174:2379,10.196.69.175:2379" + # char '/' need to escape '\/' + clusterStore="etcd:\/\/" + index=1 + etcdHostsSize=${#ETCD_HOSTS[@]} + for item in ${ETCD_HOSTS[@]} + do + clusterStore="${clusterStore}${item}:2379" + if [[ ${index} -lt ${etcdHostsSize}-1 ]]; then + clusterStore=${clusterStore}"," + fi + index=$(($index+1)) + done + echo "clusterStore=${clusterStore}" + sed -i "s/CLUSTER_STORE_REPLACE/${clusterStore}/g" $INSTALL_TEMP_DIR/docker/daemon.json >>$LOG + + sed -i "s/DOCKER_REGISTRY_REPLACE/${DOCKER_REGISTRY}/g" $INSTALL_TEMP_DIR/docker/daemon.json >>$LOG + sed -i "s/LOCAL_HOST_IP_REPLACE/${LOCAL_HOST_IP}/g" $INSTALL_TEMP_DIR/docker/daemon.json >>$LOG + sed -i "s/YARN_DNS_HOST_REPLACE/${YARN_DNS_HOST}/g" $INSTALL_TEMP_DIR/docker/daemon.json >>$LOG + sed -i "s/LOCAL_DNS_HOST_REPLACE/${LOCAL_DNS_HOST}/g" $INSTALL_TEMP_DIR/docker/daemon.json >>$LOG + + # Delete the ASF license comment in the daemon.json file, otherwise it will cause a json format error. + sed -i '1,16d' $INSTALL_TEMP_DIR/docker/daemon.json + + if [ ! -d "/etc/docker" ]; then + mkdir /etc/docker + fi + + cp $INSTALL_TEMP_DIR/docker/daemon.json /etc/docker/ + cp $INSTALL_TEMP_DIR/docker/docker.service /etc/systemd/system/ >>$LOG +} + +## @description install docker +## @audience public +## @stability stable +function install_docker() +{ + install_docker_bin + install_docker_config + + systemctl daemon-reload + systemctl enable docker.service +} + +## @description unstall docker +## @audience public +## @stability stable +function uninstall_docker() +{ + echo "stop docker service" + systemctl stop docker + + echo "remove docker" + uninstall_docker_bin + + rm /etc/docker/daemon.json >>$LOG + rm /etc/systemd/system/docker.service >>$LOG + + systemctl daemon-reload +} + +## @description start docker +## @audience public +## @stability stable +function start_docker() +{ + systemctl restart docker + systemctl status docker + docker info +} + +## @description stop docker +## @audience public +## @stability stable +function stop_docker() +{ + systemctl stop docker + systemctl status docker +} + +## @description check if the containers exist +## @audience public +## @stability stable +function containers_exist() +{ + local dockerContainersInfo=`docker ps -a --filter NAME=$1` + echo ${dockerContainersInfo} | grep $1 +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/download-server.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/download-server.sh new file mode 100644 index 0000000000..62af8279fa --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/download-server.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description start download server +## @audience public +## @stability stable +function start_download_server() +{ + if [[ "$DOWNLOAD_SERVER_IP" != "$LOCAL_HOST_IP" ]]; then + echo -e "\033[31mERROR: Only $DOWNLOAD_SERVER_IP can start the download service.\033[0m" + return 1 + fi + + echo -e "You can put the install package file in the \033[34m${DOWNLOAD_DIR}\033[0m folder first, Or automatic download." + echo -n "Do you want to start download http server?[y|n]" + read myselect + if [[ "$myselect" = "y" || "$myselect" = "Y" ]] + then + download_etcd_bin + download_calico_bin + download_docker_rpm + download_nvidia_driver + download_nvidia_docker_bin + download_yarn_container_executor + + python -m SimpleHTTPServer ${DOWNLOAD_SERVER_PORT} + fi +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/environment.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/environment.sh new file mode 100644 index 0000000000..17f59b2359 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/environment.sh @@ -0,0 +1,213 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description check operation System +## @audience public +## @stability stable +function check_operationSystem() +{ + echo -e "The submarine assembly support \033[32m[centos-release-7-3.1611.el7.centos.x86_64]\033[0m or higher operating system version." + + case ${OPERATING_SYSTEM} in + centos) + local operationSystemVersion=`rpm --query centos-release` + echo -e "The current operating system version is \e[31m[${operationSystemVersion}]\e[0m" | tee -a $LOG + ;; + *) + echo -e "\033[31mWARN: The submarine assembly Unsupported [${OPERATING_SYSTEM}] operating system\033[0m" + ;; + esac +} + +## @description update operation System Kernel +## @audience public +## @stability stable +function update_operationSystemKernel() +{ + echo "If the server is unable to connect to the network, execute the following command yourself: + wget http://vault.centos.org/7.3.1611/os/x86_64/Packages/kernel-headers-3.10.0-514.el7.x86_64.rpm + rpm -ivh kernel-headers-3.10.0-514.el7.x86_64.rpm" + + echo -n "Do you want to kernel upgrades?[y|n]" + read myselect + if [[ "$myselect" = "y" || "$myselect" = "Y" ]] + then + echo "Now try to use the yum command for kernel upgrades ..." + yum install kernel-devel-$(uname -r) kernel-headers-$(uname -r) + + local kernelVersion=`uname -r` + echo -e "After the upgrade, the operating system kernel version is \e[31m${kernelVersion}\e[0m" | tee -a $LOG + fi +} + +## @description check operation system kernel +## @audience public +## @stability stable +function check_operationSystemKernel() +{ +case ${OPERATING_SYSTEM} in +centos) + local kernelVersion=`uname -r` + + echo -e "Submarine support operating system kernel version is \033[32m 3.10.0-514.el7.x86_64 \033[0m" | tee -a $LOG + echo -e "Current operating system kernel version is \e[31m${kernelVersion}\e[0m" | tee -a $LOG + + update_operationSystemKernel + ;; +*) + echo -e "\033[31m WARN: The submarine assembly Unsupported operating system [${OPERATING_SYSTEM}] \033[0m" + ;; +esac +} + +## @description get gcc version +## @audience public +## @stability stable +function get_gcc_version() +{ + local gccVersion=`gcc --version` + version=${gccVersion%Copyright*} + echo $version +} + +## @description install gcc +## @audience public +## @stability stable +function install_gcc() +{ + echo -n "Do you want to install gcc?[y|n]" + read myselect + if [[ "$myselect" = "y" || "$myselect" = "Y" ]]; then + echo "Execute the yum install gcc make g++ command" + yum install gcc make g++ + + local gccVersion=`gcc --version` + echo -e "After the install, the gcc version is \e[31m${gccVersion}\e[0m" | tee -a $LOG + fi +} + +## @description check gcc Version +## @audience public +## @stability stable +function check_gccVersion() +{ + local gccVersionInfo=`gcc --version` + local gccVersion=${gccVersionInfo%Copyright*} + + if [[ "$gccVersion" = "" ]]; then + echo "The gcc was not installed on the system. Automated installation ..." + install_gcc + else + echo -e "Submarine gcc version need \033[34mgcc (GCC) 4.8.5 20150623 (Red Hat 4.8.5-11)\033[0m or higher." + echo -e "Current gcc version was \033[34m${gccVersion}\033[0m" + fi +} + +## @description check GPU +## @audience public +## @stability stable +function check_GPU() +{ + gpuInfo=`lspci | grep -i nvidia` + + if [[ "$gpuInfo" = "" ]]; then + echo -e "\033[31mERROR: The system did not detect the GPU graphics card.\033[0m" + else + echo -e "\033[32mINFO: The system detect the GPU graphics card.\033[0m" + fi +} + +## @description check user group +## @audience public +## @stability stable +function check_userGroup() +{ + echo -e "check hadoop user group ..." + + echo -e "Hadoop runs the required user [hdfs, mapred, yarn] and groups [hdfs, mapred, yarn, hadoop] installed by ambari." + echo -e "If you are not using ambari for hadoop installation, +then you can add the user and group by root by executing the following command: +\033[34madduser hdfs +adduser mapred +adduser yarn +addgroup hadoop +usermod -aG hdfs,hadoop hdfs +usermod -aG mapred,hadoop mapred +usermod -aG yarn,hadoop yarn +usermod -aG hdfs,hadoop hadoop +groupadd docker +usermod -aG docker yarn +usermod -aG docker hadoop\033[0m\n" + + echo -e "check docker user group ..." + # check user group + DOCKER_USER_GROUP='docker' + egrep "^${DOCKER_USER_GROUP}" /etc/group >& /dev/null + if [[ $? -ne 0 ]]; then + echo -e "user group ${DOCKER_USER_GROUP} does not exist, Please execute the following command:" + echo -e "\033[34mgroupadd $DOCKER_USER_GROUP\033[0m" + fi + + # check user + USER_GROUP=(yarn hadoop) + for user in ${USER_GROUP[@]} + do + egrep "^${user}" /etc/passwd >& /dev/null + if [[ $? -ne 0 ]]; then + echo -e "User ${user} does not exist, Please execute the following command:" + echo -e "\033[34madduser ${user}\033[0m" + echo -e "\033[34musermod -aG ${DOCKER_USER_GROUP} ${user}\033[0m" + fi + + echo -e "Please execute the following command:" + echo -e "\033[34musermod -aG ${DOCKER_USER_GROUP} ${user}\033[0m" + done +} + +## @description Some preparatory work for nvidia driver installation +## @audience public +## @stability stable +function prepare_nvidia_environment() +{ + echo "prepare nvidia environment ..." + + yum -y update + yum install kernel-devel-$(uname -r) kernel-headers-$(uname -r) + + yum -y install epel-release + yum -y install dkms + + echo -e "\033[34m ===== Please manually execute the following command ===== +# 1. Disable nouveau +# Add the content 'rd.driver.blacklist=nouveau nouveau.modeset=0' +# to the 'GRUB_CMDLINE_LINUX' configuration item in the /etc/default/grub file. +root:> vi /etc/default/grub +vi:> GRUB_CMDLINE_LINUX=\"rd.driver.blacklist=nouveau nouveau.modeset=0 ...\" + +# 2. Generate configuration +root:> grub2-mkconfig -o /boot/grub2/grub.cfg + +# 3. Open (new) /etc/modprobe.d/blacklist.conf, add content 'blacklist nouveau' +root:> vi /etc/modprobe.d/blacklist.conf +vi:> blacklist nouveau + +# 4. Update configuration and reboot +root:> mv /boot/initramfs-$(uname -r).img /boot/initramfs-$(uname -r)-nouveau.img +root:> dracut /boot/initramfs-$(uname -r).img $(uname -r) +root:> reboot +\033[0m" +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/etcd.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/etcd.sh new file mode 100644 index 0000000000..48df80ee0e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/etcd.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description download etcd bin +## @audience public +## @stability stable +function download_etcd_bin() +{ + # my download http server + if [[ -n "$DOWNLOAD_HTTP" ]]; then + MY_ETCD_DOWNLOAD_URL="${DOWNLOAD_HTTP}/downloads/etcd/${ETCD_TAR_GZ}" + else + MY_ETCD_DOWNLOAD_URL=${ETCD_DOWNLOAD_URL} + fi + + if [[ -f "${DOWNLOAD_DIR}/etcd/${ETCD_TAR_GZ}" ]]; then + echo "${DOWNLOAD_DIR}/etcd/${ETCD_TAR_GZ} is exist." + else + echo "download ${MY_ETCD_DOWNLOAD_URL} ..." + wget -P ${DOWNLOAD_DIR}/etcd ${MY_ETCD_DOWNLOAD_URL} + fi +} + +## @description install etcd bin +## @audience public +## @stability stable +function install_etcd_bin() +{ + download_etcd_bin + + # install etcd bin + mkdir -p ${INSTALL_TEMP_DIR} + rm -rf ${INSTALL_TEMP_DIR}/etcd-*-linux-amd6 + tar zxvf ${DOWNLOAD_DIR}/etcd/${ETCD_TAR_GZ} -C ${INSTALL_TEMP_DIR} + + cp -f ${INSTALL_TEMP_DIR}/etcd-*-linux-amd64/etcd /usr/bin + cp -f ${INSTALL_TEMP_DIR}/etcd-*-linux-amd64/etcdctl /usr/bin + + mkdir -p /var/lib/etcd + chmod -R a+rw /var/lib/etcd +} + +## @description install etcd config +## @audience public +## @stability stable +function install_etcd_config() +{ + # config etcd.service + rm -rf ${INSTALL_TEMP_DIR}/etcd + cp -rf ${PACKAGE_DIR}/etcd ${INSTALL_TEMP_DIR}/ + + # 1. Replace name with ETCD_NODE_NAME_REPLACE based on the location of the local IP in $ETCD_HOSTS + indexEtcdList=$(indexByEtcdHosts ${LOCAL_HOST_IP}) + # echo ${indexEtcdList} + etcdNodeName="etcdnode${indexEtcdList}" + # echo ${etcdNodeName} + sed -i "s/ETCD_NODE_NAME_REPLACE/${etcdNodeName}/g" $INSTALL_TEMP_DIR/etcd/etcd.service >>$LOG + + # 2. Replace local IP address + sed -i "s/LOCAL_HOST_REPLACE/${LOCAL_HOST_IP}/g" $INSTALL_TEMP_DIR/etcd/etcd.service >>$LOG + + # 3. Replace the initial-cluster parameter + # --initial-cluster=etcdnode1=http://10.196.69.173:2380,etcdnode2=http://10.196.69.174:2380,etcdnode3=http://10.196.69.175:2380 \ + initialCluster='' + index=0 + etcdHostsSize=${#ETCD_HOSTS[@]} + for item in ${ETCD_HOSTS[@]} + do + # char '/' need to escape '\/' + initialCluster="${initialCluster}etcdnode${index}=http:\/\/${item}:2380" + if [[ ${index} -lt ${etcdHostsSize}-1 ]]; then + initialCluster=${initialCluster}"," + fi + index=$(($index+1)) + done + #echo "initialCluster=${initialCluster}" + sed -i "s/INITIAL_CLUSTER_REPLACE/${initialCluster}/g" $INSTALL_TEMP_DIR/etcd/etcd.service >>$LOG + + cp $INSTALL_TEMP_DIR/etcd/etcd.service /etc/systemd/system/ >>$LOG +} + +## @description install etcd +## @audience public +## @stability stable +function install_etcd() +{ + index=$(indexByEtcdHosts ${LOCAL_HOST_IP}) + if [ -z "$index" ]; then + echo -e "STOP: This host\033[31m[${LOCAL_HOST_IP}]\033[0m is not in the ETCD server list\033[31m[${ETCD_HOSTS[@]}]\033[0m" + return 1 + fi + + install_etcd_bin + + install_etcd_config + + systemctl daemon-reload + systemctl enable etcd.service +} + +## @description uninstall etcd +## @audience public +## @stability stable +function uninstall_etcd() +{ + echo "stop etcd.service" + systemctl stop etcd.service + + echo "rm etcd ..." + rm /usr/bin/etcd + rm /usr/bin/etcdctl + rm -rf /var/lib/etcd + rm /etc/systemd/system/etcd.service + + systemctl daemon-reload +} + +## @description start etcd +## @audience public +## @stability stable +function start_etcd() +{ + systemctl restart etcd.service + + echo " ===== Check the status of the etcd service =====" + echo " exec etcdctl cluster-health" + etcdctl cluster-health + echo " exec etcdctl cluster-health" + etcdctl member list +} + +## @description stop etcd +## @audience public +## @stability stable +function stop_etcd() +{ + systemctl stop etcd.service +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/hadoop.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/hadoop.sh new file mode 100644 index 0000000000..9df5d7e5b2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/hadoop.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description install yarn +## @audience public +## @stability stable +function install_yarn() +{ + install_yarn_container_executor + install_yarn_config +} + +## @description uninstall yarn +## @audience public +## @stability stable +function uninstall_yarn() +{ + rm -rf /etc/yarn/sbin/Linux-amd64-64/* + rm -rf /etc/yarn/sbin/etc/hadoop/* +} + +## @description download yarn container executor +## @audience public +## @stability stable +function download_yarn_container_executor() +{ + # my download http server + if [[ -n "$DOWNLOAD_HTTP" ]]; then + MY_YARN_CONTAINER_EXECUTOR_PATH="${DOWNLOAD_HTTP}/downloads/hadoop/container-executor" + else + MY_YARN_CONTAINER_EXECUTOR_PATH=${YARN_CONTAINER_EXECUTOR_PATH} + fi + + if [ ! -d "${DOWNLOAD_DIR}/hadoop" ]; then + mkdir -p ${DOWNLOAD_DIR}/hadoop + fi + + if [[ -f "${DOWNLOAD_DIR}/hadoop/container-executor" ]]; then + echo "${DOWNLOAD_DIR}/hadoop/container-executor is exist." + else + if [[ -n "$DOWNLOAD_HTTP" ]]; then + echo "download ${MY_YARN_CONTAINER_EXECUTOR_PATH} ..." + wget -P ${DOWNLOAD_DIR}/hadoop ${MY_YARN_CONTAINER_EXECUTOR_PATH} + else + echo "copy ${MY_YARN_CONTAINER_EXECUTOR_PATH} ..." + cp ${MY_YARN_CONTAINER_EXECUTOR_PATH} ${DOWNLOAD_DIR}/hadoop/ + fi + fi +} + +## @description install yarn container executor +## @audience public +## @stability stable +function install_yarn_container_executor() +{ + echo "install yarn container executor file ..." + + download_yarn_container_executor + + if [ ! -d "/etc/yarn/sbin/Linux-amd64-64" ]; then + mkdir -p /etc/yarn/sbin/Linux-amd64-64 + fi + if [ -f "/etc/yarn/sbin/Linux-amd64-64/container-executor" ]; then + rm /etc/yarn/sbin/Linux-amd64-64/container-executor + fi + + cp -f ${DOWNLOAD_DIR}/hadoop/container-executor /etc/yarn/sbin/Linux-amd64-64 + + sudo chmod 6755 /etc/yarn/sbin/Linux-amd64-64 + sudo chown :yarn /etc/yarn/sbin/Linux-amd64-64/container-executor + sudo chmod 6050 /etc/yarn/sbin/Linux-amd64-64/container-executor +} + +## @description install yarn config +## @audience public +## @stability stable +function install_yarn_config() +{ + echo "install yarn config file ..." + + cp -R ${PACKAGE_DIR}/hadoop ${INSTALL_TEMP_DIR}/ + + find="/" + replace="\/" + escape_yarn_nodemanager_local_dirs=${YARN_NODEMANAGER_LOCAL_DIRS//$find/$replace} + escape_yarn_nodemanager_log_dirs=${YARN_NODEMANAGER_LOG_DIRS//$find/$replace} + escape_yarn_hierarchy=${YARN_HIERARCHY//$find/$replace} + + sed -i "s/YARN_NODEMANAGER_LOCAL_DIRS_REPLACE/${escape_yarn_nodemanager_local_dirs}/g" $INSTALL_TEMP_DIR/hadoop/container-executor.cfg >>$LOG + sed -i "s/YARN_NODEMANAGER_LOG_DIRS_REPLACE/${escape_yarn_nodemanager_log_dirs}/g" $INSTALL_TEMP_DIR/hadoop/container-executor.cfg >>$LOG + sed -i "s/DOCKER_REGISTRY_REPLACE/${DOCKER_REGISTRY}/g" $INSTALL_TEMP_DIR/hadoop/container-executor.cfg >>$LOG + sed -i "s/CALICO_NETWORK_NAME_REPLACE/${CALICO_NETWORK_NAME}/g" $INSTALL_TEMP_DIR/hadoop/container-executor.cfg >>$LOG + sed -i "s/YARN_HIERARCHY_REPLACE/${escape_yarn_hierarchy}/g" $INSTALL_TEMP_DIR/hadoop/container-executor.cfg >>$LOG + + # Delete the ASF license comment in the container-executor.cfg file, otherwise it will cause a cfg format error. + sed -i '1,16d' $INSTALL_TEMP_DIR/hadoop/container-executor.cfg + + if [ ! -d "/etc/yarn/sbin/etc/hadoop" ]; then + mkdir -p /etc/yarn/sbin/etc/hadoop + fi + + cp -f $INSTALL_TEMP_DIR/hadoop/container-executor.cfg /etc/yarn/sbin/etc/hadoop/ +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/menu.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/menu.sh new file mode 100644 index 0000000000..68f91c1260 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/menu.sh @@ -0,0 +1,444 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description main menu +## @audience public +## @stability stable +main_menu() +{ +cat< [prepare system environment] +------------------------------------------------------------------------------------ +MENULIST +echo -e " \e[32m1.prepare operation system\e[0m" +echo -e " \e[32m2.prepare operation system kernel\e[0m" +echo -e " \e[32m3.prepare GCC version\e[0m" +echo -e " \e[32m4.check GPU\e[0m" +echo -e " \e[32m5.prepare user&group\e[0m" +echo -e " \e[32m6.prepare nvidia environment\e[0m" +echo -e "" +echo -e " \e[32mb.back main menu\e[0m" +cat< [install component] +------------------------------------------------------------------------------------ +MENULIST +echo -e " \e[32m1.instll etcd\e[0m" +echo -e " \e[32m2.instll docker\e[0m" +echo -e " \e[32m3.instll calico network\e[0m" +echo -e " \e[32m4.instll nvidia driver\e[0m" +echo -e " \e[32m5.instll nvidia docker\e[0m" +echo -e " \e[32m6.instll yarn container-executor\e[0m" +echo -e " \e[32m7.instll submarine autorun script\e[0m" +echo -e "" +echo -e " \e[32mb.back main menu\e[0m" +cat< [uninstll component] +------------------------------------------------------------------------------------ +MENULIST +echo -e " \e[32m1.uninstll etcd\e[0m" +echo -e " \e[32m2.uninstll docker\e[0m" +echo -e " \e[32m3.uninstll calico network\e[0m" +echo -e " \e[32m4.uninstll nvidia driver\e[0m" +echo -e " \e[32m5.uninstll nvidia docker\e[0m" +echo -e " \e[32m6.uninstll yarn container-executor\e[0m" +echo -e " \e[32m7.uninstll submarine autorun script\e[0m" +echo -e "" +echo -e " \e[32mb.back main menu\e[0m" +cat< [stop component] +------------------------------------------------------------------------------------ +MENULIST +echo -e " \e[32m1.start etcd\e[0m" +echo -e " \e[32m2.start docker\e[0m" +echo -e " \e[32m3.start calico network\e[0m" +echo -e "" +echo -e " \e[32mb.back main menu\e[0m" +cat< [stop component] +------------------------------------------------------------------------------------ +MENULIST +echo -e " \e[32m1.stop etcd\e[0m" +echo -e " \e[32m2.stop docker\e[0m" +echo -e " \e[32m3.stop calico network\e[0m" +echo -e "" +echo -e " \e[32mb.back main menu\e[0m" +cat< nvidia-docker run --rm ${DOCKER_REGISTRY}/nvidia/cuda:9.0-devel nvidia-smi +# If you don't see the list of graphics cards above, the NVIDIA driver installation failed. ===== +\033[0m" + + echo -e "\033[32m===== Please manually execute the following command =====\033[0m" + echo -e "\033[32m# Test with tf.test.is_gpu_available() +shell:> nvidia-docker run -it ${DOCKER_REGISTRY}/tensorflow/tensorflow:1.9.0-gpu bash +# In docker container +container:> python +python:> import tensorflow as tf +python:> tf.test.is_gpu_available() +python:> exit() +\033[0m" +} + +## @description uninstall nvidia docker +## @audience public +## @stability stable +function uninstall_nvidia_docker() +{ + echo "This method is not implemented." +} + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia.sh new file mode 100644 index 0000000000..7b2b1de0d5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +nvidia_run_file="" + +## @description get nvidia version +## @audience public +## @stability stable +function get_nvidia_version() +{ + chmod +x ${DOWNLOAD_DIR}/nvidia/nvidia-detect + local nvidia_detect_info=`${DOWNLOAD_DIR}/nvidia/nvidia-detect -v` + echo $nvidia_detect_info | sed "s/^.*This device requires the current \([0-9.]*\).*/\1/" +} + +## @description download nvidia driver +## @audience public +## @stability stable +function download_nvidia_driver() +{ + # submarin http server + if [[ -n "$DOWNLOAD_HTTP" ]]; then + MY_NVIDIA_DETECT_URL="${DOWNLOAD_HTTP}/downloads/nvidia/nvidia-detect" + else + MY_NVIDIA_DETECT_URL=${NVIDIA_DETECT_URL} + fi + + # download nvidia_detect + if [[ -f "${DOWNLOAD_DIR}/nvidia/nvidia-detect" ]]; then + echo "${DOWNLOAD_DIR}/nvidia/nvidia-detect is exist." + else + echo "download ${MY_NVIDIA_DETECT_URL} ..." + wget -P ${DOWNLOAD_DIR}/nvidia/ ${MY_NVIDIA_DETECT_URL} + fi + + echo "execution nvidia-detect to check the graphics card ..." + local nvidiaVersion=`get_nvidia_version` + echo -e "detect nvidia version is \033[31m${nvidiaVersion}\033[0m" + + # download NVIDIA driver + if [[ "$nvidiaVersion" = "" ]]; then + echo -e "\033[31mERROR: No graphics card device detected.\033[0m" + return 1 + else + nvidia_run_file="NVIDIA-Linux-x86_64-${nvidiaVersion}.run" + + # submarin http server + if [[ -n "$DOWNLOAD_HTTP" ]]; then + MY_NVIDIA_DRIVER_RUN_URL="${DOWNLOAD_HTTP}/downloads/nvidia/${nvidia_run_file}" + else + # http://us.download.nvidia.com/XFree86/Linux-x86_64/390.87/NVIDIA-Linux-x86_64-390.87.run + MY_NVIDIA_DRIVER_RUN_URL="http://us.download.nvidia.com/XFree86/Linux-x86_64/${nvidiaVersion}/${nvidia_run_file}" + fi + + if [[ -f ${DOWNLOAD_DIR}/nvidia/${nvidia_run_file} ]]; then + echo "NVIDIA driver files already exist in the ${DOWNLOAD_DIR}/nvidia/${nvidia_run_file} directory." + echo "===== Please make sure the ${DOWNLOAD_DIR}/nvidia/nvidia/${nvidia_run_file} file is complete and can be used normally. =====" + else + echo "Download the NVIDIA driver from the ${MY_NVIDIA_DRIVER_RUN_URL}" + wget -P ${DOWNLOAD_DIR}/nvidia/ ${MY_NVIDIA_DRIVER_RUN_URL} + fi + fi +} + +## @description install nvidia +## @audience public +## @stability stable +function install_nvidia() +{ + download_nvidia_driver + + # Confirm that the system disables nouveau + local disable_nouveau_info=`lsmod | grep nouveau` + if [[ "$disable_nouveau_info" = "" ]]; then + echo "===== Start installing the NVIDIA driver =====" + echo -e "Some options during the installation +Would you like to register the kernel module sources with DKMS? + This will allow DKMS to automatically build a new module, if you install a different kernel later. \033[33m[Yes]\033[0m +Install NVIDIA's 32-bit compatibility libraries \033[33m[Yes]\033[0m +centos Install NVIDIA's 32-bit compatibility libraries \033[33m[Yes]\033[0m +Would you like to run the nvidia-xconfig utility to automatically update your X configuration file... \033[33m[No]\033[0m" + sleep 2 + sh ${DOWNLOAD_DIR}/nvidia/${nvidia_run_file} + else + echo -e "ERROR: Nouveau is not disabled" + return 1 + fi + + echo -e "\033[32m===== execute nvidia-smi. You should be able to see the list of graphics cards =====\033[0m" + sleep 1 + nvidia-smi +} + +## @description uninstall nvidia +## @audience public +## @stability stable +function uninstall_nvidia() +{ + if [ ! -f "/usr/bin/nvidia-uninstall" ]; then + echo -e "\033[31mERROR: /usr/bin/nvidia-uninstall file is not exist!\033[0m" + return 1 + fi + + echo -e "execute /usr/bin/nvidia-uninstall" + /usr/bin/nvidia-uninstall +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/submarine.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/submarine.sh new file mode 100644 index 0000000000..b73d5e77c2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/submarine.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description install submarine autorun script +## @audience public +## @stability stable +function install_submarine() +{ + cp ${PACKAGE_DIR}/submarine/submarine.sh /etc/rc.d/init.d/submarine.sh + chmod +x /etc/rc.d/init.d/submarine.sh + chmod +x /etc/rc.d/rc.local + + if [ `grep -c "/etc/rc.d/init.d/submarine.sh" /etc/rc.d/rc.local` -eq '0' ]; then + echo "/etc/rc.d/init.d/submarine.sh">> /etc/rc.d/rc.local + fi +} + +## @description uninstall submarine autorun script +## @audience public +## @stability stable +function uninstall_submarine() +{ + rm /etc/rc.d/init.d/submarine.sh +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh new file mode 100644 index 0000000000..7b3c2a94ce --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/utils.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## @description check install user +## @audience public +## @stability stable +function check_install_user() +{ + if [[ $(id -u) -ne 0 ]];then + echo "This script must be run with a ROOT user!" + exit # don't call exit_install() + fi +} + +## @description exit install +## @audience public +## @stability stable +function exit_install() +{ + echo "Exit the installation!" | tee -a $LOG + exit $1 +} + +## @description Check if the IP address format is correct +## @audience public +## @stability stable +function valid_ip() +{ + local ip=$1 + local stat=1 + + if [[ $ip =~ ^[0-9]{1,3\}.[0-9]{1,3\}.[0-9]{1,3\}.[0-9]{1,3\}$ ]]; then + OIFS=$IFS + IFS='.' + ip=($ip) + IFS=$OIFS + + if [[ ${ip[0]} -le 255 && ${ip[1]} -le 255 && ${ip[2]} -le 255 && ${ip[3]} -le 255 ]]; then + stat=$? + fi + fi + + return $stat +} + +## @description Check if the configuration file configuration is correct +## @audience public +## @stability stable +function check_install_conf() +{ + echo "Check if the configuration file configuration is correct ..." | tee -a $LOG + + # check etcd conf + hostCount=${#ETCD_HOSTS[@]} + if [[ $hostCount -lt 3 && hostCount -ne 0 ]]; then # <>2 + echo "Number of nodes = [$hostCount], must be configured to be greater than or equal to 3 servers! " | tee -a $LOG + exit_install + fi + for ip in ${ETCD_HOSTS[@]} + do + if ! valid_ip $ip; then + echo "]ETCD_HOSTS=[$ip], IP address format is incorrect! " | tee -a $LOG + exit_install + fi + done + echo "Check if the configuration file configuration is correct [ Done ]" | tee -a $LOG +} + +## @description index by EtcdHosts list +## @audience public +## @stability stable +function indexByEtcdHosts() { + index=0 + while [ "$index" -lt "${#ETCD_HOSTS[@]}" ]; do + if [ "${ETCD_HOSTS[$index]}" = "$1" ]; then + echo $index + return + fi + let "index++" + done + echo "" +} + +## @description get local IP +## @audience public +## @stability stable +function getLocalIP() +{ + local _ip _myip _line _nl=$'\n' + while IFS=$': \t' read -a _line ;do + [ -z "${_line%inet}" ] && + _ip=${_line[${#_line[1]}>4?1:2]} && + [ "${_ip#127.0.0.1}" ] && _myip=$_ip + done< <(LANG=C /sbin/ifconfig) + printf ${1+-v} $1 "%s${_nl:0:$[${#1}>0?0:1]}" $_myip +} + +## @description get ip list +## @audience public +## @stability stable +function get_ip_list() +{ + array=$(ifconfig | grep inet | grep -v inet6 | grep -v 127 | sed 's/^[ \t]*//g' | cut -d ' ' -f2) + + for ip in ${array[@]} + do + LOCAL_HOST_IP_LIST+=(${ip}) + done +}