hadoop/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs

#!/usr/bin/env bash

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

MYNAME="${BASH_SOURCE-$0}"
HADOOP_SHELL_EXECNAME="${MYNAME##*/}"

## @description  build up the hdfs command's usage text.
## @audience     public
## @stability    stable
## @replaceable  no
function hadoop_usage
{
  hadoop_add_option "--buildpaths" "attempt to add class files from build tree"
  hadoop_add_option "--daemon (start|status|stop)" "operate on a daemon"
  hadoop_add_option "--hostnames list[,of,host,names]" "hosts to use in worker mode"
  hadoop_add_option "--loglevel level" "set the log4j level for this command"
  hadoop_add_option "--hosts filename" "list of hosts to use in worker mode"
  hadoop_add_option "--workers" "turn on worker mode"

  hadoop_add_subcommand "balancer" "run a cluster balancing utility"
  hadoop_add_subcommand "cacheadmin" "configure the HDFS cache"
  hadoop_add_subcommand "classpath" "prints the class path needed to get the hadoop jar and the required libraries"
  hadoop_add_subcommand "crypto" "configure HDFS encryption zones"
  hadoop_add_subcommand "datanode" "run a DFS datanode"
  hadoop_add_subcommand "debug" "run a Debug Admin to execute HDFS debug commands"
  hadoop_add_subcommand "dfs" "run a filesystem command on the file system"
  hadoop_add_subcommand "dfsadmin" "run a DFS admin client"
  hadoop_add_subcommand "diskbalancer" "Distributes data evenly among disks on a given node"
  hadoop_add_subcommand "envvars" "display computed Hadoop environment variables"
  hadoop_add_subcommand "erasurecode" "run a HDFS ErasureCoding CLI"
  hadoop_add_subcommand "fetchdt" "fetch a delegation token from the NameNode"
  hadoop_add_subcommand "fsck" "run a DFS filesystem checking utility"
  hadoop_add_subcommand "getconf" "get config values from configuration"
  hadoop_add_subcommand "groups" "get the groups which users belong to"
  hadoop_add_subcommand "haadmin" "run a DFS HA admin client"
  hadoop_add_subcommand "jmxget" "get JMX exported values from NameNode or DataNode."
  hadoop_add_subcommand "journalnode" "run the DFS journalnode"
  hadoop_add_subcommand "lsSnapshottableDir" "list all snapshottable dirs owned by the current user"
  hadoop_add_subcommand "mover" "run a utility to move block replicas across storage types"
  hadoop_add_subcommand "namenode" "run the DFS namenode"
  hadoop_add_subcommand "nfs3" "run an NFS version 3 gateway"
  hadoop_add_subcommand "oev" "apply the offline edits viewer to an edits file"
  hadoop_add_subcommand "oiv" "apply the offline fsimage viewer to an fsimage"
  hadoop_add_subcommand "oiv_legacy" "apply the offline fsimage viewer to a legacy fsimage"
  hadoop_add_subcommand "portmap" "run a portmap service"
  hadoop_add_subcommand "secondarynamenode" "run the DFS secondary namenode"
  hadoop_add_subcommand "snapshotDiff" "diff two snapshots of a directory or diff the current directory contents with a snapshot"
  hadoop_add_subcommand "storagepolicies" "list/get/set block storage policies"
  hadoop_add_subcommand "version" "print the version"
  hadoop_add_subcommand "zkfc" "run the ZK Failover Controller daemon"
  hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" false
}

## @description  Default command handler for hadoop command
## @audience     public
## @stability    stable
## @replaceable  no
## @param        CLI arguments
function hdfscmd_case
{
  subcmd=$1
  shift

  case ${subcmd} in
    balancer)
      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
      HADOOP_CLASSNAME=org.apache.hadoop.hdfs.server.balancer.Balancer
      hadoop_debug "Appending HADOOP_BALANCER_OPTS onto HADOOP_OPTS"
      HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_BALANCER_OPTS}"
    ;;
    cacheadmin)
      HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.CacheAdmin
    ;;
    classpath)
      hadoop_do_classpath_subcommand HADOOP_CLASSNAME "$@"
    ;;
    crypto)
      HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.CryptoAdmin
    ;;
    datanode)
      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
      # Determine if we're starting a secure datanode, and
      # if so, redefine appropriate variables
      if [[ -n "${HADOOP_SECURE_DN_USER}" ]]; then
        HADOOP_SUBCMD_SECURESERVICE="true"
        HADOOP_SUBCMD_SECUREUSER="${HADOOP_SECURE_DN_USER}"

        # backward compatiblity
        HADOOP_SECURE_PID_DIR="${HADOOP_SECURE_PID_DIR:-$HADOOP_SECURE_DN_PID_DIR}"
        HADOOP_SECURE_LOG_DIR="${HADOOP_SECURE_LOG_DIR:-$HADOOP_SECURE_DN_LOG_DIR}"

        hadoop_debug "Appending HADOOP_DATANODE_OPTS onto HADOOP_OPTS"
        hadoop_debug "Appending HADOOP_DN_SECURE_EXTRA_OPTS onto HADOOP_OPTS"
        HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_DATANODE_OPTS} ${HADOOP_DN_SECURE_EXTRA_OPTS}"
        HADOOP_CLASSNAME="org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter"
      else
        hadoop_debug "Appending HADOOP_DATANODE_OPTS onto HADOOP_OPTS"
        HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_DATANODE_OPTS}"
        HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.datanode.DataNode'
      fi
    ;;
    debug)
      HADOOP_CLASSNAME='org.apache.hadoop.hdfs.tools.DebugAdmin'
    ;;
    dfs)
      HADOOP_CLASSNAME=org.apache.hadoop.fs.FsShell
      hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
      HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
    ;;
    dfsadmin)
      HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DFSAdmin
      hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
      HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
    ;;
    diskbalancer)
      HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DiskBalancer
      hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
      HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
    ;;
    envvars)
      echo "JAVA_HOME='${JAVA_HOME}'"
      echo "HADOOP_HDFS_HOME='${HADOOP_HDFS_HOME}'"
      echo "HDFS_DIR='${HDFS_DIR}'"
      echo "HDFS_LIB_JARS_DIR='${HDFS_LIB_JARS_DIR}'"
      echo "HADOOP_CONF_DIR='${HADOOP_CONF_DIR}'"
      echo "HADOOP_TOOLS_HOME='${HADOOP_TOOLS_HOME}'"
      echo "HADOOP_TOOLS_DIR='${HADOOP_TOOLS_DIR}'"
      echo "HADOOP_TOOLS_LIB_JARS_DIR='${HADOOP_TOOLS_LIB_JARS_DIR}'"
      exit 0
    ;;
    erasurecode)
      HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.erasurecode.ECCli
      hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
      HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
    ;;
    fetchdt)
      HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DelegationTokenFetcher
    ;;
    fsck)
      HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DFSck
      hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
      HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
    ;;
    getconf)
      HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.GetConf
    ;;
    groups)
      HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.GetGroups
    ;;
    haadmin)
      HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DFSHAAdmin
      hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
      HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
    ;;
    journalnode)
      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
      HADOOP_CLASSNAME='org.apache.hadoop.hdfs.qjournal.server.JournalNode'
      hadoop_debug "Appending HADOOP_JOURNALNODE_OPTS onto HADOOP_OPTS"
      HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_JOURNALNODE_OPTS}"
    ;;
    jmxget)
      HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.JMXGet
    ;;
    lsSnapshottableDir)
      HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.snapshot.LsSnapshottableDir
    ;;
    mover)
      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
      HADOOP_CLASSNAME=org.apache.hadoop.hdfs.server.mover.Mover
      hadoop_debug "Appending HADOOP_MOVER_OPTS onto HADOOP_OPTS"
      HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_MOVER_OPTS}"
    ;;
    namenode)
      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
      HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.namenode.NameNode'
      hadoop_debug "Appending HADOOP_NAMENODE_OPTS onto HADOOP_OPTS"
      HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NAMENODE_OPTS}"
      hadoop_add_param HADOOP_OPTS hdfs.audit.logger "-Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER}"
    ;;
    nfs3)
      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
      if [[ -n "${HADOOP_PRIVILEGED_NFS_USER}" ]]; then
        HADOOP_SUBCMD_SECURESERVICE="true"
        HADOOP_SUBCMD_SECUREUSER="${HADOOP_PRIVILEGED_NFS_USER}"

        # backward compatiblity
        HADOOP_SECURE_PID_DIR="${HADOOP_SECURE_PID_DIR:-$HADOOP_SECURE_NFS3_PID_DIR}"
        HADOOP_SECURE_LOG_DIR="${HADOOP_SECURE_LOG_DIR:-$HADOOP_SECURE_NFS3_LOG_DIR}"

        hadoop_debug "Appending HADOOP_NFS3_OPTS onto HADOOP_OPTS"
        hadoop_debug "Appending HADOOP_NFS3_SECURE_EXTRA_OPTS onto HADOOP_OPTS"
        HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NFS3_OPTS} ${HADOOP_NFS3_SECURE_EXTRA_OPTS}"
        HADOOP_CLASSNAME=org.apache.hadoop.hdfs.nfs.nfs3.PrivilegedNfsGatewayStarter
      else
        hadoop_debug "Appending HADOOP_NFS3_OPTS onto HADOOP_OPTS"
        HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NFS3_OPTS}"
        HADOOP_CLASSNAME=org.apache.hadoop.hdfs.nfs.nfs3.Nfs3
      fi
    ;;
    oev)
      HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer
    ;;
    oiv)
      HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewerPB
    ;;
    oiv_legacy)
      HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewer
    ;;
    portmap)
      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
      HADOOP_CLASSNAME=org.apache.hadoop.portmap.Portmap
      hadoop_debug "Appending HADOOP_PORTMAP_OPTS onto HADOOP_OPTS"
      HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_PORTMAP_OPTS}"
    ;;
    secondarynamenode)
      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
      HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode'
      hadoop_debug "Appending HADOOP_SECONDARYNAMENODE_OPTS onto HADOOP_OPTS"
      HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_SECONDARYNAMENODE_OPTS}"
      hadoop_add_param HADOOP_OPTS hdfs.audit.logger "-Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER}"
    ;;
    snapshotDiff)
      HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff
    ;;
    storagepolicies)
      HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.StoragePolicyAdmin
    ;;
    version)
      HADOOP_CLASSNAME=org.apache.hadoop.util.VersionInfo
    ;;
    zkfc)
      HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
      HADOOP_CLASSNAME='org.apache.hadoop.hdfs.tools.DFSZKFailoverController'
      hadoop_debug "Appending HADOOP_ZKFC_OPTS onto HADOOP_OPTS"
      HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_ZKFC_OPTS}"
    ;;
    *)
      HADOOP_CLASSNAME="${subcmd}"
      if ! hadoop_validate_classname "${HADOOP_CLASSNAME}"; then
        hadoop_exit_with_usage 1
      fi
    ;;
  esac
}

# let's locate libexec...
if [[ -n "${HADOOP_HOME}" ]]; then
  HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
else
  bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P)
  HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
fi

HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}"
# shellcheck disable=SC2034
HADOOP_NEW_CONFIG=true
if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then
  . "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh"
else
  echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1
  exit 1
fi

if [[ $# = 0 ]]; then
  hadoop_exit_with_usage 1
fi

HADOOP_SUBCMD=$1
shift

HADOOP_SUBCMD_ARGS=("$@")

if declare -f hdfs_subcommand_"${HADOOP_SUBCMD}" >/dev/null 2>&1; then
  hadoop_debug "Calling dynamically: hdfs_subcommand_${HADOOP_SUBCMD} ${HADOOP_SUBCMD_ARGS[*]}"
  "hdfs_subcommand_${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}"
else
  hdfscmd_case "${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}"
fi

hadoop_verify_user "${HADOOP_SUBCMD}"

if [[ ${HADOOP_WORKER_MODE} = true ]]; then
  hadoop_common_worker_mode_execute "${HADOOP_HDFS_HOME}/bin/hdfs" "${HADOOP_USER_PARAMS[@]}"
  exit $?
fi

if [[ "${HADOOP_SUBCMD_SECURESERVICE}" = true ]]; then
  HADOOP_SECURE_USER="${HADOOP_SUBCMD_SECUREUSER}"
  hadoop_verify_secure_prereq
  hadoop_setup_secure_service
  priv_outfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out"
  priv_errfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.err"
  priv_pidfile="${HADOOP_PID_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid"
  daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out"
  daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid"
else
  daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out"
  daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid"
fi

if [[ "${HADOOP_DAEMON_MODE}" != "default" ]]; then
  # shellcheck disable=SC2034
  HADOOP_ROOT_LOGGER="${HADOOP_DAEMON_ROOT_LOGGER}"
  if [[ -n "${HADOOP_SUBCMD_SECURESERVICE}" ]]; then
    # shellcheck disable=SC2034
    HADOOP_LOGFILE="hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.log"
  else
    # shellcheck disable=SC2034
    HADOOP_LOGFILE="hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.log"
  fi
fi

hadoop_finalize

if [[ "${HADOOP_SUBCMD_SUPPORTDAEMONIZATION}" = true ]]; then
  if [[ "${HADOOP_SUBCMD_SECURESERVICE}" = true ]]; then
    hadoop_secure_daemon_handler \
      "${HADOOP_DAEMON_MODE}" \
      "${HADOOP_SUBCMD}" \
      "${HADOOP_CLASSNAME}" \
      "${daemon_pidfile}" \
      "${daemon_outfile}" \
      "${priv_pidfile}" \
      "${priv_outfile}" \
      "${priv_errfile}" \
      "${HADOOP_SUBCMD_ARGS[@]}"
  else
    hadoop_daemon_handler \
      "${HADOOP_DAEMON_MODE}" \
      "${HADOOP_SUBCMD}" \
      "${HADOOP_CLASSNAME}" \
      "${daemon_pidfile}" \
      "${daemon_outfile}" \
      "${HADOOP_SUBCMD_ARGS[@]}"
  fi
  exit $?
else
  # shellcheck disable=SC2086
  hadoop_java_exec "${HADOOP_SUBCMD}" "${HADOOP_CLASSNAME}" "${HADOOP_SUBCMD_ARGS[@]}"
fi