big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
2cf4d638af
hadoop/dev-support/bin/hadoop.sh

634 lines
18 KiB
Bash
Executable File
Raw Blame History

#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# SHELLDOC-IGNORE
#
# Override these to match Apache Hadoop's requirements
personality_plugins "all,-ant,-gradle,-scalac,-scaladoc"
# These flags are needed to run Yetus against Hadoop on Windows.
WINDOWS_FLAGS=(
"-Pnative-win"
"-Dhttps.protocols=TLSv1.2"
"-Drequire.openssl"
"-Drequire.test.libhadoop"
"-Dshell-executable=${BASH_EXECUTABLE}"
"-Dopenssl.prefix=${VCPKG_INSTALLED_PACKAGES}"
"-Dcmake.prefix.path=${VCPKG_INSTALLED_PACKAGES}"
"-Dwindows.cmake.toolchain.file=${CMAKE_TOOLCHAIN_FILE}"
"-Dwindows.cmake.build.type=RelWithDebInfo"
"-Dwindows.build.hdfspp.dll=off"
"-Dwindows.no.sasl=on"
"-Duse.platformToolsetVersion=v142"
)
## @description Globals specific to this personality
## @audience private
## @stability evolving
function personality_globals
{
# shellcheck disable=SC2034
BUILDTOOL=maven
#shellcheck disable=SC2034
PATCH_BRANCH_DEFAULT=trunk
#shellcheck disable=SC2034
PATCH_NAMING_RULE="https://cwiki.apache.org/confluence/display/HADOOP/How+To+Contribute"
#shellcheck disable=SC2034
JIRA_ISSUE_RE='^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$'
#shellcheck disable=SC2034
GITHUB_REPO_DEFAULT="apache/hadoop"
HADOOP_HOMEBREW_DIR=${HADOOP_HOMEBREW_DIR:-$(brew --prefix 2>/dev/null)}
if [[ -z "${HADOOP_HOMEBREW_DIR}" ]]; then
HADOOP_HOMEBREW_DIR=/usr/local
fi
}
function personality_parse_args
{
declare i
for i in "$@"; do
case ${i} in
--hadoop-isal-prefix=*)
delete_parameter "${i}"
ISAL_HOME=${i#*=}
;;
--hadoop-openssl-prefix=*)
delete_parameter "${i}"
OPENSSL_HOME=${i#*=}
;;
--hadoop-snappy-prefix=*)
delete_parameter "${i}"
SNAPPY_HOME=${i#*=}
;;
esac
done
}
## @description Calculate the actual module ordering
## @audience private
## @stability evolving
## @param ordering
function hadoop_order
{
declare ordering=$1
declare hadoopm
if [[ ${ordering} = normal ]]; then
hadoopm="${CHANGED_MODULES[*]}"
elif [[ ${ordering} = union ]]; then
hadoopm="${CHANGED_UNION_MODULES}"
elif [[ ${ordering} = mvnsrc ]]; then
hadoopm="${MAVEN_SRC_MODULES[*]}"
elif [[ ${ordering} = mvnsrctest ]]; then
hadoopm="${MAVEN_SRCTEST_MODULES[*]}"
else
hadoopm="${ordering}"
fi
echo "${hadoopm}"
}
## @description Retrieves the Hadoop project version defined in the root pom.xml
## @audience private
## @stability evolving
## @returns 0 on success, 1 on failure
function load_hadoop_version
{
if [[ -f "${BASEDIR}/pom.xml" ]]; then
HADOOP_VERSION=$(grep '<version>' "${BASEDIR}/pom.xml" \
| head -1 \
| "${SED}" -e 's|^ *<version>||' -e 's|</version>.*$||' \
| cut -f1 -d- )
return 0
else
return 1
fi
}
## @description Determine if it is safe to run parallel tests
## @audience private
## @stability evolving
## @param ordering
function hadoop_test_parallel
{
if load_hadoop_version; then
export HADOOP_VERSION
else
return 1
fi
hmajor=${HADOOP_VERSION%%\.*}
hmajorminor=${HADOOP_VERSION%\.*}
hminor=${hmajorminor##*\.}
# ... and just for reference
#hmicro=${HADOOP_VERSION##*\.}
# Apache Hadoop v2.8.0 was the first one to really
# get working parallel unit tests
if [[ ${hmajor} -lt 3 && ${hminor} -lt 8 ]]; then
return 1
fi
return 0
}
## @description Install extra modules for unit tests
## @audience private
## @stability evolving
## @param ordering
function hadoop_unittest_prereqs
{
declare input=$1
declare mods
declare need_common=0
declare building_common=0
declare module
declare flags
declare fn
# prior to running unit tests, hdfs needs libhadoop.so built
# if we're building root, then this extra work is moot
#shellcheck disable=SC2086
mods=$(hadoop_order ${input})
for module in ${mods}; do
if [[ ${module} = hadoop-hdfs-project* ]]; then
need_common=1
elif [[ ${module} = hadoop-common-project/hadoop-common
|| ${module} = hadoop-common-project ]]; then
building_common=1
elif [[ ${module} = . ]]; then
return
fi
done
# Windows builds *ALWAYS* need hadoop-common compiled
case ${OSTYPE} in
Windows_NT|CYGWIN*|MINGW*|MSYS*)
need_common=1
;;
esac
if [[ ${need_common} -eq 1
&& ${building_common} -eq 0 ]]; then
echo "unit test pre-reqs:"
module="hadoop-common-project/hadoop-common"
fn=$(module_file_fragment "${module}")
flags="$(hadoop_native_flags) $(yarn_ui2_flag)"
pushd "${BASEDIR}/${module}" >/dev/null || return 1
# shellcheck disable=SC2086
echo_and_redirect "${PATCH_DIR}/maven-unit-prereq-${fn}-install.txt" \
"${MAVEN}" "${MAVEN_ARGS[@]}" install -DskipTests ${flags}
popd >/dev/null || return 1
fi
}
## @description Calculate the flags/settings for yarn-ui v2 build
## @description based upon the OS
## @audience private
## @stability evolving
function yarn_ui2_flag
{
if [[ ${BUILD_NATIVE} != true ]]; then
return
fi
# Now it only tested on Linux/OSX, don't enable the profile on
# windows until it get verified
case ${OSTYPE} in
Linux)
# shellcheck disable=SC2086
echo -Pyarn-ui
;;
Darwin)
echo -Pyarn-ui
;;
*)
# Do nothing
;;
esac
}
## @description Calculate the flags/settings for native code
## @description based upon the OS
## @audience private
## @stability evolving
function hadoop_native_flags
{
if [[ ${BUILD_NATIVE} != true ]]; then
return
fi
declare -a args
# Based upon HADOOP-11937
#
# Some notes:
#
# - getting fuse to compile on anything but Linux
# is always tricky.
# - Darwin assumes homebrew is in use.
# - HADOOP-12027 required for bzip2 on OS X.
# - bzip2 is broken in lots of places
# (the shared library is considered experimental)
# e.g, HADOOP-12027 for OS X. so no -Drequire.bzip2
#
args=("-Drequire.test.libhadoop")
if [[ -d "${ISAL_HOME}/include" ]]; then
args=("${args[@]}" "-Disal.prefix=${ISAL_HOME}")
fi
if [[ -d "${OPENSSL_HOME}/include" ]]; then
args=("${args[@]}" "-Dopenssl.prefix=${OPENSSL_HOME}")
elif [[ -d "${HADOOP_HOMEBREW_DIR}/opt/openssl/" ]]; then
args=("${args[@]}" "-Dopenssl.prefix=${HADOOP_HOMEBREW_DIR}/opt/openssl/")
fi
if [[ -d "${SNAPPY_HOME}/include" ]]; then
args=("${args[@]}" "-Dsnappy.prefix=${SNAPPY_HOME}")
elif [[ -d "${HADOOP_HOMEBREW_DIR}/include/snappy.h" ]]; then
args=("${args[@]}" "-Dsnappy.prefix=${HADOOP_HOMEBREW_DIR}/opt/snappy")
fi
case ${OSTYPE} in
Linux)
# shellcheck disable=SC2086
echo \
-Pnative \
-Drequire.fuse \
-Drequire.openssl \
-Drequire.snappy \
-Drequire.valgrind \
-Drequire.zstd \
"${args[@]}"
;;
Darwin)
echo \
"${args[@]}" \
-Pnative \
-Drequire.snappy \
-Drequire.openssl
;;
Windows_NT|CYGWIN*|MINGW*|MSYS*)
echo \
"${args[@]}" \
-Drequire.snappy \
-Pdist \
-Dtar \
"${WINDOWS_FLAGS[@]}"
;;
*)
echo \
"${args[@]}"
;;
esac
}
## @description Queue up modules for this personality
## @audience private
## @stability evolving
## @param repostatus
## @param testtype
function personality_modules
{
declare repostatus=$1
declare testtype=$2
declare extra=""
declare ordering="normal"
declare needflags=false
declare foundbats=false
declare flags
declare fn
declare i
declare hadoopm
yetus_debug "Personality: ${repostatus} ${testtype}"
clear_personality_queue
case ${testtype} in
asflicense)
# this is very fast and provides the full path if we do it from
# the root of the source
personality_enqueue_module .
return
;;
checkstyle)
ordering="union"
extra="-DskipTests"
;;
compile)
ordering="union"
extra="-DskipTests"
needflags=true
# if something in common changed, we build the whole world
if [[ "${CHANGED_MODULES[*]}" =~ hadoop-common ]]; then
yetus_debug "hadoop personality: javac + hadoop-common = ordering set to . "
ordering="."
fi
;;
distclean)
ordering="."
extra="-DskipTests"
;;
javadoc)
if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then
ordering=.
fi
if [[ "${repostatus}" = patch && "${BUILDMODE}" = patch ]]; then
echo "javadoc pre-reqs:"
for i in hadoop-project \
hadoop-common-project/hadoop-annotations; do
fn=$(module_file_fragment "${i}")
pushd "${BASEDIR}/${i}" >/dev/null || return 1
echo "cd ${i}"
echo_and_redirect "${PATCH_DIR}/maven-${fn}-install.txt" \
"${MAVEN}" "${MAVEN_ARGS[@]}" install
popd >/dev/null || return 1
done
fi
extra="-Pdocs -DskipTests"
;;
mvneclipse)
if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then
ordering=.
fi
;;
mvninstall)
extra="-DskipTests"
if [[ "${repostatus}" = branch || "${BUILDMODE}" = full ]]; then
ordering=.
fi
;;
mvnsite)
if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then
ordering=.
fi
;;
unit)
if [[ "$IS_WINDOWS" && "$IS_WINDOWS" == 1 && (-z "$IS_NIGHTLY_BUILD" || "$IS_NIGHTLY_BUILD" == 0) ]]; then
echo "Won't run unit tests for Windows in pre-commit CI"
return
fi
extra="-Dsurefire.rerunFailingTestsCount=2"
if [[ "${BUILDMODE}" = full ]]; then
ordering=mvnsrc
elif [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then
ordering=.
fi
if [[ ${TEST_PARALLEL} = "true" ]] ; then
if hadoop_test_parallel; then
extra="${extra} -Pparallel-tests"
if [[ -n ${TEST_THREADS:-} ]]; then
extra="${extra} -DtestsThreadCount=${TEST_THREADS}"
fi
fi
fi
needflags=true
hadoop_unittest_prereqs "${ordering}"
if ! verify_needed_test javac; then
yetus_debug "hadoop: javac not requested"
if ! verify_needed_test native; then
yetus_debug "hadoop: native not requested"
yetus_debug "hadoop: adding -DskipTests to unit test"
extra="-DskipTests"
fi
fi
for i in "${CHANGED_FILES[@]}"; do
if [[ "${i}" =~ \.bats ]]; then
foundbats=true
fi
done
if ! verify_needed_test shellcheck && [[ ${foundbats} = false ]]; then
yetus_debug "hadoop: NO shell code change detected; disabling shelltest profile"
extra="${extra} -P!shelltest"
else
extra="${extra} -Pshelltest"
fi
;;
*)
extra="-DskipTests"
;;
esac
if [[ ${needflags} = true ]]; then
flags="$(hadoop_native_flags) $(yarn_ui2_flag)"
extra="${extra} ${flags}"
fi
if [[ "$IS_WINDOWS" && "$IS_WINDOWS" == 1 ]]; then
extra="-Ptest-patch -Pdist -Dtar ${WINDOWS_FLAGS[*]} ${extra}"
fi
for module in $(hadoop_order ${ordering}); do
# shellcheck disable=SC2086
personality_enqueue_module ${module} ${extra}
done
}
## @description Add tests based upon personality needs
## @audience private
## @stability evolving
## @param filename
function personality_file_tests
{
declare filename=$1
yetus_debug "Using Hadoop-specific personality_file_tests"
if [[ ${filename} =~ src/main/webapp ]]; then
yetus_debug "tests/webapp: ${filename}"
add_test shadedclient
elif [[ ${filename} =~ \.sh
|| ${filename} =~ \.cmd
|| ${filename} =~ src/scripts
|| ${filename} =~ src/test/scripts
|| ${filename} =~ src/main/bin
|| ${filename} =~ shellprofile\.d
|| ${filename} =~ src/main/conf
]]; then
yetus_debug "tests/shell: ${filename}"
add_test mvnsite
add_test unit
elif [[ ${filename} =~ \.md$
|| ${filename} =~ \.md\.vm$
|| ${filename} =~ src/site
]]; then
yetus_debug "tests/site: ${filename}"
add_test mvnsite
elif [[ ${filename} =~ \.c$
|| ${filename} =~ \.cc$
|| ${filename} =~ \.h$
|| ${filename} =~ \.hh$
|| ${filename} =~ \.proto$
|| ${filename} =~ \.cmake$
|| ${filename} =~ CMakeLists.txt
]]; then
yetus_debug "tests/units: ${filename}"
add_test compile
add_test cc
add_test mvnsite
add_test javac
add_test unit
elif [[ ${filename} =~ build.xml$
|| ${filename} =~ pom.xml$
|| ${filename} =~ \.java$
|| ${filename} =~ src/main
]]; then
yetus_debug "tests/javadoc+units: ${filename}"
add_test compile
add_test javac
add_test javadoc
add_test mvninstall
add_test mvnsite
add_test unit
add_test shadedclient
fi
# if we change anything in here, e.g. the test scripts
# then run the client artifact tests
if [[ ${filename} =~ hadoop-client-modules ]]; then
add_test shadedclient
fi
if [[ ${filename} =~ src/test ]]; then
yetus_debug "tests: src/test"
add_test unit
fi
if [[ ${filename} =~ \.java$ ]]; then
add_test spotbugs
fi
}
## @description Image to print on success
## @audience private
## @stability evolving
function hadoop_console_success
{
printf "IF9fX19fX19fX18gCjwgU3VjY2VzcyEgPgogLS0tLS0tLS0tLSAKIFwgICAg";
printf "IC9cICBfX18gIC9cCiAgXCAgIC8vIFwvICAgXC8gXFwKICAgICAoKCAgICBP";
printf "IE8gICAgKSkKICAgICAgXFwgLyAgICAgXCAvLwogICAgICAgXC8gIHwgfCAg";
printf "XC8gCiAgICAgICAgfCAgfCB8ICB8ICAKICAgICAgICB8ICB8IHwgIHwgIAog";
printf "ICAgICAgIHwgICBvICAgfCAgCiAgICAgICAgfCB8ICAgfCB8ICAKICAgICAg";
printf "ICB8bXwgICB8bXwgIAo"
}
###################################################
# Hadoop project specific check of IT for shaded artifacts
add_test_type shadedclient
## @description check for test modules and add test/plugins as needed
## @audience private
## @stability evolving
function shadedclient_initialize
{
maven_add_install shadedclient
}
## @description build client facing shaded and non-shaded artifacts and test them
## @audience private
## @stability evolving
## @param repostatus
function shadedclient_rebuild
{
declare repostatus=$1
declare logfile="${PATCH_DIR}/${repostatus}-shadedclient.txt"
declare module
declare -a modules=()
yetus_debug "hadoop personality: seeing if we need the test of client artifacts."
for module in hadoop-client-modules/hadoop-client-check-invariants \
hadoop-client-modules/hadoop-client-check-test-invariants \
hadoop-client-modules/hadoop-client-integration-tests; do
if [ -d "${module}" ]; then
yetus_debug "hadoop personality: test module '${module}' is present."
modules+=(-pl "${module}")
fi
done
if [ ${#modules[@]} -eq 0 ]; then
echo "hadoop personality: no test modules present, skipping check of client artifacts."
return 0
fi
big_console_header "Checking client artifacts on ${repostatus} with shaded clients"
extra=(
"-Dtest=NoUnitTests"
"-Dmaven.javadoc.skip=true"
"-Dcheckstyle.skip=true"
"-Dspotbugs.skip=true"
)
if [[ "$IS_WINDOWS" && "$IS_WINDOWS" == 1 ]]; then
# shellcheck disable=SC2206
extra+=(${WINDOWS_FLAGS[*]})
# The shaded client integration tests require the Hadoop jars that were just built to be
# installed in the local maven repository.
# shellcheck disable=SC2086
echo_and_redirect "${logfile}" \
"${MAVEN}" "${MAVEN_ARGS[@]}" install -fae --batch-mode \
-DskipTests -DskipDocs -Pdist -Dtar ${extra[*]}
# The shaded client integration tests spawn a MiniDFS and MiniYARN cluster for testing. Both of
# them require winutils.exe to be found in the PATH and HADOOP_HOME to be set.
if load_hadoop_version; then
export HADOOP_HOME="${SOURCEDIR}/hadoop-dist/target/hadoop-${HADOOP_VERSION}-SNAPSHOT"
WIN_HADOOP_HOME=$(cygpath -w -a "${HADOOP_HOME}")
export PATH="${PATH};${WIN_HADOOP_HOME}\bin"
else
yetus_error "[WARNING] Unable to extract the Hadoop version and thus HADOOP_HOME is not set. Some tests may fail."
fi
fi
# shellcheck disable=SC2086
echo_and_redirect "${logfile}" \
"${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am "${modules[@]}" ${extra[*]}
big_console_header "Checking client artifacts on ${repostatus} with non-shaded clients"
# shellcheck disable=SC2086
echo_and_redirect "${logfile}" \
"${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am \
"${modules[@]}" \
-DskipShade -Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true \
-Dspotbugs.skip=true ${extra[*]}
count=$("${GREP}" -c '\[ERROR\]' "${logfile}")
if [[ ${count} -gt 0 ]]; then
add_vote_table -1 shadedclient "${repostatus} has errors when building and testing our client artifacts."
return 1
fi
add_vote_table +1 shadedclient "${repostatus} has no errors when building and testing our client artifacts."
return 0
}
Reference in New Issue View Git Blame Copy Permalink