#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. MYNAME="${BASH_SOURCE-$0}" function hadoop_usage { hadoop_add_option "--buildpaths" "attempt to add class files from build tree" hadoop_add_option "--daemon (start|status|stop)" "operate on a daemon" hadoop_add_option "--hostnames list[,of,host,names]" "hosts to use in slave mode" hadoop_add_option "--loglevel level" "set the log4j level for this command" hadoop_add_option "--hosts filename" "list of hosts to use in slave mode" hadoop_add_option "--slaves" "turn on slave mode" hadoop_add_subcommand "balancer" "run a cluster balancing utility" hadoop_add_subcommand "cacheadmin" "configure the HDFS cache" hadoop_add_subcommand "classpath" "prints the class path needed to get the hadoop jar and the required libraries" hadoop_add_subcommand "crypto" "configure HDFS encryption zones" hadoop_add_subcommand "datanode" "run a DFS datanode" hadoop_add_subcommand "debug" "run a Debug Admin to execute HDFS debug commands" hadoop_add_subcommand "dfs" "run a filesystem command on the file system" hadoop_add_subcommand "dfsadmin" "run a DFS admin client" hadoop_add_subcommand "envvars" "display computed Hadoop environment variables" hadoop_add_subcommand "erasurecode" "run a HDFS ErasureCoding CLI" hadoop_add_subcommand "fetchdt" "fetch a delegation token from the NameNode" hadoop_add_subcommand "fsck" "run a DFS filesystem checking utility" hadoop_add_subcommand "getconf" "get config values from configuration" hadoop_add_subcommand "groups" "get the groups which users belong to" hadoop_add_subcommand "haadmin" "run a DFS HA admin client" hadoop_add_subcommand "jmxget" "get JMX exported values from NameNode or DataNode." hadoop_add_subcommand "journalnode" "run the DFS journalnode" hadoop_add_subcommand "lsSnapshottableDir" "list all snapshottable dirs owned by the current user" hadoop_add_subcommand "mover" "run a utility to move block replicas across storage types" hadoop_add_subcommand "namenode" "run the DFS namenode" hadoop_add_subcommand "nfs3" "run an NFS version 3 gateway" hadoop_add_subcommand "oev" "apply the offline edits viewer to an edits file" hadoop_add_subcommand "oiv" "apply the offline fsimage viewer to an fsimage" hadoop_add_subcommand "oiv_legacy" "apply the offline fsimage viewer to a legacy fsimage" hadoop_add_subcommand "portmap" "run a portmap service" hadoop_add_subcommand "secondarynamenode" "run the DFS secondary namenode" hadoop_add_subcommand "snapshotDiff" "diff two snapshots of a directory or diff the current directory contents with a snapshot" hadoop_add_subcommand "storagepolicies" "list/get/set block storage policies" hadoop_add_subcommand "version" "print the version" hadoop_add_subcommand "zkfc" "run the ZK Failover Controller daemon" hadoop_generate_usage "${MYNAME}" false } # let's locate libexec... if [[ -n "${HADOOP_PREFIX}" ]]; then HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" else bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P) HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec" fi HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}" # shellcheck disable=SC2034 HADOOP_NEW_CONFIG=true if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then . "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" else echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1 exit 1 fi if [[ $# = 0 ]]; then hadoop_exit_with_usage 1 fi COMMAND=$1 shift case ${COMMAND} in balancer) supportdaemonization="true" CLASS=org.apache.hadoop.hdfs.server.balancer.Balancer hadoop_debug "Appending HADOOP_BALANCER_OPTS onto HADOOP_OPTS" HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_BALANCER_OPTS}" ;; cacheadmin) CLASS=org.apache.hadoop.hdfs.tools.CacheAdmin ;; classpath) hadoop_do_classpath_subcommand CLASS "$@" ;; crypto) CLASS=org.apache.hadoop.hdfs.tools.CryptoAdmin ;; datanode) supportdaemonization="true" # Determine if we're starting a secure datanode, and # if so, redefine appropriate variables if [[ -n "${HADOOP_SECURE_DN_USER}" ]]; then secure_service="true" secure_user="${HADOOP_SECURE_DN_USER}" # backward compatiblity HADOOP_SECURE_PID_DIR="${HADOOP_SECURE_PID_DIR:-$HADOOP_SECURE_DN_PID_DIR}" HADOOP_SECURE_LOG_DIR="${HADOOP_SECURE_LOG_DIR:-$HADOOP_SECURE_DN_LOG_DIR}" hadoop_debug "Appending HADOOP_DATANODE_OPTS onto HADOOP_OPTS" hadoop_debug "Appending HADOOP_DN_SECURE_EXTRA_OPTS onto HADOOP_OPTS" HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_DATANODE_OPTS} ${HADOOP_DN_SECURE_EXTRA_OPTS}" CLASS="org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter" else hadoop_debug "Appending HADOOP_DATANODE_OPTS onto HADOOP_OPTS" HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_DATANODE_OPTS}" CLASS='org.apache.hadoop.hdfs.server.datanode.DataNode' fi ;; debug) CLASS='org.apache.hadoop.hdfs.tools.DebugAdmin' ;; dfs) CLASS=org.apache.hadoop.fs.FsShell hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS" HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}" ;; dfsadmin) CLASS=org.apache.hadoop.hdfs.tools.DFSAdmin hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS" HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}" ;; envvars) echo "JAVA_HOME='${JAVA_HOME}'" echo "HADOOP_HDFS_HOME='${HADOOP_HDFS_HOME}'" echo "HDFS_DIR='${HDFS_DIR}'" echo "HDFS_LIB_JARS_DIR='${HDFS_LIB_JARS_DIR}'" echo "HADOOP_CONF_DIR='${HADOOP_CONF_DIR}'" echo "HADOOP_TOOLS_PATH='${HADOOP_TOOLS_PATH}'" exit 0 ;; erasurecode) CLASS=org.apache.hadoop.hdfs.tools.erasurecode.ECCli hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS" HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}" ;; fetchdt) CLASS=org.apache.hadoop.hdfs.tools.DelegationTokenFetcher ;; fsck) CLASS=org.apache.hadoop.hdfs.tools.DFSck hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS" HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}" ;; getconf) CLASS=org.apache.hadoop.hdfs.tools.GetConf ;; groups) CLASS=org.apache.hadoop.hdfs.tools.GetGroups ;; haadmin) CLASS=org.apache.hadoop.hdfs.tools.DFSHAAdmin hadoop_add_to_classpath_toolspath hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS" HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}" ;; journalnode) supportdaemonization="true" CLASS='org.apache.hadoop.hdfs.qjournal.server.JournalNode' hadoop_debug "Appending HADOOP_JOURNALNODE_OPTS onto HADOOP_OPTS" HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_JOURNALNODE_OPTS}" ;; jmxget) CLASS=org.apache.hadoop.hdfs.tools.JMXGet ;; lsSnapshottableDir) CLASS=org.apache.hadoop.hdfs.tools.snapshot.LsSnapshottableDir ;; mover) supportdaemonization="true" CLASS=org.apache.hadoop.hdfs.server.mover.Mover hadoop_debug "Appending HADOOP_MOVER_OPTS onto HADOOP_OPTS" HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_MOVER_OPTS}" ;; namenode) supportdaemonization="true" CLASS='org.apache.hadoop.hdfs.server.namenode.NameNode' hadoop_debug "Appending HADOOP_NAMENODE_OPTS onto HADOOP_OPTS" HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NAMENODE_OPTS}" hadoop_add_param HADOOP_OPTS hdfs.audit.logger "-Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER}" ;; nfs3) supportdaemonization="true" if [[ -n "${HADOOP_PRIVILEGED_NFS_USER}" ]]; then secure_service="true" secure_user="${HADOOP_PRIVILEGED_NFS_USER}" # backward compatiblity HADOOP_SECURE_PID_DIR="${HADOOP_SECURE_PID_DIR:-$HADOOP_SECURE_NFS3_PID_DIR}" HADOOP_SECURE_LOG_DIR="${HADOOP_SECURE_LOG_DIR:-$HADOOP_SECURE_NFS3_LOG_DIR}" hadoop_debug "Appending HADOOP_NFS3_OPTS onto HADOOP_OPTS" hadoop_debug "Appending HADOOP_NFS3_SECURE_EXTRA_OPTS onto HADOOP_OPTS" HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NFS3_OPTS} ${HADOOP_NFS3_SECURE_EXTRA_OPTS}" CLASS=org.apache.hadoop.hdfs.nfs.nfs3.PrivilegedNfsGatewayStarter else hadoop_debug "Appending HADOOP_NFS3_OPTS onto HADOOP_OPTS" HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NFS3_OPTS}" CLASS=org.apache.hadoop.hdfs.nfs.nfs3.Nfs3 fi ;; oev) CLASS=org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer ;; oiv) CLASS=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewerPB ;; oiv_legacy) CLASS=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewer ;; portmap) supportdaemonization="true" CLASS=org.apache.hadoop.portmap.Portmap hadoop_debug "Appending HADOOP_PORTMAP_OPTS onto HADOOP_OPTS" HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_PORTMAP_OPTS}" ;; secondarynamenode) supportdaemonization="true" CLASS='org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode' hadoop_debug "Appending HADOOP_SECONDARYNAMENODE_OPTS onto HADOOP_OPTS" HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_SECONDARYNAMENODE_OPTS}" hadoop_add_param HADOOP_OPTS hdfs.audit.logger "-Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER}" ;; snapshotDiff) CLASS=org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff ;; storagepolicies) CLASS=org.apache.hadoop.hdfs.tools.StoragePolicyAdmin ;; version) CLASS=org.apache.hadoop.util.VersionInfo ;; zkfc) supportdaemonization="true" CLASS='org.apache.hadoop.hdfs.tools.DFSZKFailoverController' hadoop_debug "Appending HADOOP_ZKFC_OPTS onto HADOOP_OPTS" HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_ZKFC_OPTS}" ;; *) CLASS="${COMMAND}" if ! hadoop_validate_classname "${CLASS}"; then hadoop_exit_with_usage 1 fi ;; esac hadoop_verify_user "${COMMAND}" if [[ ${HADOOP_SLAVE_MODE} = true ]]; then hadoop_common_slave_mode_execute "${HADOOP_HDFS_HOME}/bin/hdfs" "${HADOOP_USER_PARAMS[@]}" exit $? fi if [[ -n "${secure_service}" ]]; then HADOOP_SECURE_USER="${secure_user}" hadoop_verify_secure_prereq hadoop_setup_secure_service priv_outfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out" priv_errfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.err" priv_pidfile="${HADOOP_PID_DIR}/privileged-${HADOOP_IDENT_STRING}-${COMMAND}.pid" daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out" daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${COMMAND}.pid" else daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out" daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}.pid" fi if [[ "${HADOOP_DAEMON_MODE}" != "default" ]]; then # shellcheck disable=SC2034 HADOOP_ROOT_LOGGER="${HADOOP_DAEMON_ROOT_LOGGER}" if [[ -n "${secure_service}" ]]; then # shellcheck disable=SC2034 HADOOP_LOGFILE="hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.log" else # shellcheck disable=SC2034 HADOOP_LOGFILE="hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.log" fi fi hadoop_finalize if [[ -n "${supportdaemonization}" ]]; then if [[ -n "${secure_service}" ]]; then hadoop_secure_daemon_handler \ "${HADOOP_DAEMON_MODE}" "${COMMAND}" "${CLASS}"\ "${daemon_pidfile}" "${daemon_outfile}" \ "${priv_pidfile}" "${priv_outfile}" "${priv_errfile}" "$@" else hadoop_daemon_handler "${HADOOP_DAEMON_MODE}" "${COMMAND}" "${CLASS}"\ "${daemon_pidfile}" "${daemon_outfile}" "$@" fi exit $? else # shellcheck disable=SC2086 hadoop_java_exec "${COMMAND}" "${CLASS}" "$@" fi