hadoop/dev-support/bin/dist-copynativelibs
Liang-Chi Hsieh 8f60a90688 HADOOP-17125. Use snappy-java in SnappyCodec (#2297)
This switches the SnappyCodec to use the java-snappy codec, rather than the native one.

To use the codec, snappy-java.jar (from org.xerial.snappy) needs to be on the classpath.

This comesin as an avro dependency,  so it is already on the hadoop-common classpath,
as well as in hadoop-common/lib.
The version used is now managed in the hadoop-project POM; initially 1.1.7.7

Contributed by DB Tsai and Liang-Chi Hsieh

Change-Id: Id52a404a0005480e68917cd17f0a27b7744aea4e
2020-10-06 17:15:17 +01:00

181 lines
4.6 KiB
Bash
Executable File

#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o pipefail
# Bundle a native library if requested. Exit 1 in case error happens.
# Usage: bundle_native_lib bundleoption liboption libpattern libdir
function bundle_native_lib()
{
declare bundleoption="$1"
declare liboption="$2"
declare libpattern="$3"
declare libdir="$4"
echo "Checking to bundle with:"
echo "bundleoption=${bundleoption}, liboption=${liboption}, pattern=${libpattern} libdir=${libdir}"
if [[ "${bundleoption}" != "true" ]]; then
return
fi
if [[ -z "${libdir}" ]] || [[ ! -d "${libdir}" ]]; then
echo "The required option ${liboption} isn't given or invalid. Bundling the lib failed"
exit 1
fi
cd "${libdir}" || exit 1
${TAR} ./*"${libpattern}"* | (cd "${TARGET_DIR}"/ || exit 1; ${UNTAR})
if [[ $? -ne 0 ]]; then
echo "Bundling library with ${liboption} failed "
exit 1
fi
}
function bundle_native_bin
{
declare bundleoption="$1"
declare libbundle="$2"
declare binoption="$3"
declare binpattern="$4"
declare libdir="$5"
echo "Checking to bundle with:"
echo "bundleoption=${bundleoption}, libbundle=${libbundle}, binoption=${binoption}, libdir=${libdir}, binpattern=${binpattern}"
if [[ "${bundleoption}" != "true" ]]; then
return
fi
if [[ "${libbundle}" != "true" ]]; then
return
fi
if [[ -z "${libdir}" ]] || [[ ! -d "${libdir}" ]]; then
echo "The required option ${liboption} isn't given or invalid. Bundling the lib failed"
exit 1
fi
cd "${libdir}" || exit 1
${TAR} ./*"${binpattern}"* | (cd "${TARGET_BIN_DIR}"/ || exit 1 ; ${UNTAR})
if [[ $? -ne 0 ]]; then
echo "Bundling bin files for ${binoption} failed"
exit 1
fi
}
for i in "$@"; do
case "${i}" in
--version=*)
VERSION=${i#*=}
;;
--artifactid=*)
ARTIFACTID=${i#*=}
;;
--builddir=*)
BUILD_DIR=${i#*=}
;;
--isallib=*)
ISALLIB=${i#*=}
;;
--isalbundle=*)
ISALBUNDLE=${i#*=}
;;
--pmdklib=*)
PMDKLIB=${i#*=}
;;
--pmdkbundle=*)
PMDKBUNDLE=${i#*=}
;;
--opensslbinbundle=*)
OPENSSLBINBUNDLE=${i#*=}
;;
--openssllib=*)
OPENSSLLIB=${i#*=}
;;
--openssllibbundle=*)
OPENSSLLIBBUNDLE=${i#*=}
;;
--snappylib=*)
SNAPPYLIB=${i#*=}
;;
--snappylibbundle=*)
SNAPPYLIBBUNDLE=${i#*=}
;;
--zstdbinbundle=*)
ZSTDBINBUNDLE=${i#*=}
;;
--zstdlib=*)
ZSTDLIB=${i#*=}
;;
--zstdlibbundle=*)
ZSTDLIBBUNDLE=${i#*=}
;;
esac
done
TAR='tar cf -'
UNTAR='tar xfBp -'
LIB_DIR="${BUILD_DIR}/native/target/usr/local/lib"
BIN_DIR="${BUILD_DIR}/bin"
TARGET_DIR="${BUILD_DIR}/${ARTIFACTID}-${VERSION}/lib/native"
TARGET_BIN_DIR="${BUILD_DIR}/${ARTIFACTID}-${VERSION}/bin"
# Most systems
if [[ -d "${LIB_DIR}" ]]; then
mkdir -p "${TARGET_DIR}"
cd "${LIB_DIR}" || exit 1
${TAR} lib* | (cd "${TARGET_DIR}"/ || exit 1; ${UNTAR})
if [[ $? -ne 0 ]]; then
echo "Bundling lib files failed"
exit 1
fi
bundle_native_lib "${SNAPPYLIBBUNDLE}" "snappy.lib" "libsnappy." "${SNAPPYLIB}"
bundle_native_lib "${ZSTDLIBBUNDLE}" "zstd.lib" "libzstd." "${ZSTDLIB}"
bundle_native_lib "${OPENSSLLIBBUNDLE}" "openssl.lib" "libcrypto." "${OPENSSLLIB}"
bundle_native_lib "${ISALBUNDLE}" "isal.lib" "libisal." "${ISALLIB}"
bundle_native_lib "${PMDKBUNDLE}" "pmdk.lib" "pmdk" "${PMDKLIB}"
fi
# Windows
# Windows doesn't have a LIB_DIR, everything goes into bin
if [[ -d "${BIN_DIR}" ]] ; then
mkdir -p "${TARGET_BIN_DIR}"
cd "${BIN_DIR}" || exit 1
${TAR} ./* | (cd "${TARGET_BIN_DIR}"/ || exit 1; ${UNTAR})
if [[ $? -ne 0 ]]; then
echo "Bundling bin files failed"
exit 1
fi
bundle_native_bin "${ZSTDBINBUNDLE}" "${ZSTDLIBBUNDLE}" "zstd.lib" "zstd" "${ZSTDLIB}"
bundle_native_bin "${OPENSSLBINBUNDLE}" "${OPENSSLLIBBUNDLE}" "openssl.lib" "crypto" "${OPENSSLLIB}"
fi