HDFS-8635. Migrate HDFS native build to new CMake framework (Alan Burlison via Colin P. McCabe)
This commit is contained in:
Colin Patrick Mccabe
parent
9ee7b6e6c4
commit
29df73dc05
@ -691,6 +691,9 @@ Release 2.8.0 - UNRELEASED
|
||||
HDFS-8493. Consolidate truncate() related implementation in a single class.
|
||||
(Rakesh R via wheat9)
|
||||
|
||||
HDFS-8635. Migrate HDFS native build to new CMake framework (Alan Burlison
|
||||
via Colin P. McCabe)
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
|
||||
|
||||
@ -18,106 +18,59 @@
|
||||
|
||||
cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
|
||||
|
||||
# Default to release builds
|
||||
set(CMAKE_BUILD_TYPE, Release)
|
||||
list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../../../hadoop-common-project/hadoop-common)
|
||||
include(HadoopCommon)
|
||||
|
||||
include(../../../hadoop-common-project/hadoop-common/src/JNIFlags.cmake NO_POLICY_SCOPE)
|
||||
#
|
||||
# Main configuration
|
||||
#
|
||||
|
||||
# Compile a library with both shared and static variants
|
||||
function(add_dual_library LIBNAME)
|
||||
add_library(${LIBNAME} SHARED ${ARGN})
|
||||
add_library(${LIBNAME}_static STATIC ${ARGN})
|
||||
# Linux builds traditionally ship a libhdfs.a (static linking) and libhdfs.so
|
||||
# (dynamic linking). On Windows, we cannot use the same base name for both
|
||||
# static and dynamic, because Windows does not use distinct file extensions
|
||||
# for a statically linked library vs. a DLL import library. Both use the
|
||||
# .lib extension. On Windows, we'll build the static library as
|
||||
# hdfs_static.lib.
|
||||
if (NOT WIN32)
|
||||
set_target_properties(${LIBNAME}_static PROPERTIES OUTPUT_NAME ${LIBNAME})
|
||||
endif (NOT WIN32)
|
||||
endfunction(add_dual_library)
|
||||
|
||||
# Link both a static and a dynamic target against some libraries
|
||||
function(target_link_dual_libraries LIBNAME)
|
||||
target_link_libraries(${LIBNAME} ${ARGN})
|
||||
target_link_libraries(${LIBNAME}_static ${ARGN})
|
||||
endfunction(target_link_dual_libraries)
|
||||
|
||||
function(output_directory TGT DIR)
|
||||
SET_TARGET_PROPERTIES(${TGT} PROPERTIES
|
||||
RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
|
||||
SET_TARGET_PROPERTIES(${TGT} PROPERTIES
|
||||
ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
|
||||
SET_TARGET_PROPERTIES(${TGT} PROPERTIES
|
||||
LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
|
||||
endfunction(output_directory TGT DIR)
|
||||
|
||||
function(dual_output_directory TGT DIR)
|
||||
output_directory(${TGT} "${DIR}")
|
||||
output_directory(${TGT}_static "${DIR}")
|
||||
endfunction(dual_output_directory TGT DIR)
|
||||
|
||||
# Flatten a list into a string.
|
||||
function(FLATTEN_LIST INPUT SEPARATOR OUTPUT)
|
||||
string (REPLACE ";" "${SEPARATOR}" _TMPS "${INPUT}")
|
||||
set (${OUTPUT} "${_TMPS}" PARENT_SCOPE)
|
||||
endfunction()
|
||||
# The caller must specify where the generated headers have been placed.
|
||||
if(NOT GENERATED_JAVAH)
|
||||
message(FATAL_ERROR "You must set the CMake variable GENERATED_JAVAH")
|
||||
endif()
|
||||
|
||||
# Check to see if our compiler and linker support the __thread attribute.
|
||||
# On Linux and some other operating systems, this is a more efficient
|
||||
# alternative to POSIX thread local storage.
|
||||
INCLUDE(CheckCSourceCompiles)
|
||||
CHECK_C_SOURCE_COMPILES("int main(void) { static __thread int i = 0; return 0; }" HAVE_BETTER_TLS)
|
||||
include(CheckCSourceCompiles)
|
||||
check_c_source_compiles("int main(void) { static __thread int i = 0; return 0; }" HAVE_BETTER_TLS)
|
||||
|
||||
# Check to see if we have Intel SSE intrinsics.
|
||||
CHECK_C_SOURCE_COMPILES("#include <emmintrin.h>\nint main(void) { __m128d sum0 = _mm_set_pd(0.0,0.0); return 0; }" HAVE_INTEL_SSE_INTRINSICS)
|
||||
check_c_source_compileS("#include <emmintrin.h>\nint main(void) { __m128d sum0 = _mm_set_pd(0.0,0.0); return 0; }" HAVE_INTEL_SSE_INTRINSICS)
|
||||
|
||||
# Check if we need to link dl library to get dlopen.
|
||||
# dlopen on Linux is in separate library but on FreeBSD its in libc
|
||||
INCLUDE(CheckLibraryExists)
|
||||
CHECK_LIBRARY_EXISTS(dl dlopen "" NEED_LINK_DL)
|
||||
include(CheckLibraryExists)
|
||||
check_library_exists(dl dlopen "" NEED_LINK_DL)
|
||||
|
||||
find_package(JNI REQUIRED)
|
||||
if (NOT GENERATED_JAVAH)
|
||||
# Must identify where the generated headers have been placed
|
||||
MESSAGE(FATAL_ERROR "You must set the CMake variable GENERATED_JAVAH")
|
||||
endif (NOT GENERATED_JAVAH)
|
||||
|
||||
if (WIN32)
|
||||
if(WIN32)
|
||||
# Set the optimizer level.
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /O2")
|
||||
|
||||
# Set warning level 4.
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /W4")
|
||||
|
||||
# Skip "unreferenced formal parameter".
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4100")
|
||||
|
||||
# Skip "conditional expression is constant".
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4127")
|
||||
|
||||
# Skip deprecated POSIX function warnings.
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_CRT_NONSTDC_NO_DEPRECATE")
|
||||
|
||||
# Skip CRT non-secure function warnings. If we can convert usage of
|
||||
# strerror, getenv and ctime to their secure CRT equivalents, then we can
|
||||
# re-enable the CRT non-secure function warnings.
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_CRT_SECURE_NO_WARNINGS")
|
||||
|
||||
# Omit unneeded headers.
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DWIN32_LEAN_AND_MEAN")
|
||||
|
||||
set(OS_DIR main/native/libhdfs/os/windows)
|
||||
set(OUT_DIR target/bin)
|
||||
else (WIN32)
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wall -O2")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT -D_GNU_SOURCE")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64")
|
||||
else()
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fvisibility=hidden")
|
||||
set(OS_DIR main/native/libhdfs/os/posix)
|
||||
set(OS_LINK_LIBRARIES pthread)
|
||||
set(OUT_DIR target/usr/local/lib)
|
||||
endif (WIN32)
|
||||
endif()
|
||||
|
||||
# Configure JNI.
|
||||
include(HadoopJNI)
|
||||
|
||||
add_definitions(-DLIBHDFS_DLL_EXPORT)
|
||||
|
||||
@ -132,9 +85,9 @@ include_directories(
|
||||
)
|
||||
|
||||
set(_FUSE_DFS_VERSION 0.1.0)
|
||||
CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
|
||||
configure_file(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
|
||||
|
||||
add_dual_library(hdfs
|
||||
hadoop_add_dual_library(hdfs
|
||||
main/native/libhdfs/exception.c
|
||||
main/native/libhdfs/jni_helper.c
|
||||
main/native/libhdfs/hdfs.c
|
||||
@ -142,17 +95,17 @@ add_dual_library(hdfs
|
||||
${OS_DIR}/mutexes.c
|
||||
${OS_DIR}/thread_local_storage.c
|
||||
)
|
||||
if (NEED_LINK_DL)
|
||||
if(NEED_LINK_DL)
|
||||
set(LIB_DL dl)
|
||||
endif(NEED_LINK_DL)
|
||||
endif()
|
||||
|
||||
target_link_dual_libraries(hdfs
|
||||
hadoop_target_link_dual_libraries(hdfs
|
||||
${JAVA_JVM_LIBRARY}
|
||||
${LIB_DL}
|
||||
${OS_LINK_LIBRARIES}
|
||||
)
|
||||
|
||||
dual_output_directory(hdfs ${OUT_DIR})
|
||||
hadoop_dual_output_directory(hdfs ${OUT_DIR})
|
||||
set(LIBHDFS_VERSION "0.0.0")
|
||||
set_target_properties(hdfs PROPERTIES
|
||||
SOVERSION ${LIBHDFS_VERSION})
|
||||
@ -232,25 +185,21 @@ target_link_libraries(test_htable
|
||||
|
||||
# Skip vecsum on Windows. This could be made to work in the future by
|
||||
# introducing an abstraction layer over the sys/mman.h functions.
|
||||
if (NOT WIN32)
|
||||
add_executable(test_libhdfs_vecsum
|
||||
main/native/libhdfs/test/vecsum.c
|
||||
)
|
||||
if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
|
||||
if(NOT WIN32)
|
||||
add_executable(test_libhdfs_vecsum main/native/libhdfs/test/vecsum.c)
|
||||
if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
|
||||
target_link_libraries(test_libhdfs_vecsum
|
||||
hdfs
|
||||
pthread)
|
||||
else()
|
||||
target_link_libraries(test_libhdfs_vecsum
|
||||
hdfs
|
||||
pthread
|
||||
)
|
||||
else (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
|
||||
target_link_libraries(test_libhdfs_vecsum
|
||||
hdfs
|
||||
pthread
|
||||
rt
|
||||
)
|
||||
endif (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
|
||||
endif(NOT WIN32)
|
||||
rt)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
IF(REQUIRE_LIBWEBHDFS)
|
||||
if(REQUIRE_LIBWEBHDFS)
|
||||
add_subdirectory(contrib/libwebhdfs)
|
||||
ENDIF(REQUIRE_LIBWEBHDFS)
|
||||
endif()
|
||||
add_subdirectory(main/native/fuse-dfs)
|
||||
|
||||
@ -19,12 +19,12 @@
|
||||
find_package(CURL REQUIRED)
|
||||
|
||||
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH}
|
||||
"${CMAKE_SOURCE_DIR}/contrib/libwebhdfs/resources/")
|
||||
"${CMAKE_SOURCE_DIR}/contrib/libwebhdfs/resources/")
|
||||
|
||||
find_package(Jansson REQUIRED)
|
||||
include_directories(${JANSSON_INCLUDE_DIR})
|
||||
|
||||
add_dual_library(webhdfs
|
||||
hadoop_add_dual_library(webhdfs
|
||||
src/hdfs_web.c
|
||||
src/hdfs_http_client.c
|
||||
src/hdfs_http_query.c
|
||||
@ -35,13 +35,13 @@ add_dual_library(webhdfs
|
||||
../../${OS_DIR}/thread_local_storage.c
|
||||
../../main/native/libhdfs/common/htable.c
|
||||
)
|
||||
target_link_dual_libraries(webhdfs
|
||||
hadoop_target_link_dual_libraries(webhdfs
|
||||
${JAVA_JVM_LIBRARY}
|
||||
${CURL_LIBRARY}
|
||||
${JANSSON_LIBRARY}
|
||||
pthread
|
||||
)
|
||||
dual_output_directory(webhdfs target)
|
||||
hadoop_dual_output_directory(webhdfs target)
|
||||
set(LIBWEBHDFS_VERSION "0.0.0")
|
||||
set_target_properties(webhdfs PROPERTIES
|
||||
SOVERSION ${LIBWEBHDFS_VERSION})
|
||||
|
||||
@ -24,7 +24,8 @@
|
||||
# JANSSON_DEFINITIONS - Compiler switches required for using Jansson
|
||||
|
||||
find_path(JANSSON_INCLUDE_DIR jansson.h
|
||||
/usr/incluce
|
||||
/usr/include
|
||||
/usr/include/jansson
|
||||
/usr/local/include )
|
||||
|
||||
find_library(JANSSON_LIBRARY NAMES jansson
|
||||
|
||||
@ -18,25 +18,31 @@
|
||||
|
||||
set(CMAKE_SKIP_RPATH TRUE)
|
||||
|
||||
# Flatten a list into a string.
|
||||
function(flatten_list INPUT SEPARATOR OUTPUT)
|
||||
string (REPLACE ";" "${SEPARATOR}" _TMPS "${INPUT}")
|
||||
set (${OUTPUT} "${_TMPS}" PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
# Find Linux FUSE
|
||||
IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
|
||||
if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
|
||||
find_package(PkgConfig REQUIRED)
|
||||
pkg_check_modules(FUSE fuse)
|
||||
IF(FUSE_FOUND)
|
||||
FLATTEN_LIST("${FUSE_CFLAGS}" " " FUSE_CFLAGS)
|
||||
FLATTEN_LIST("${FUSE_LDFLAGS}" " " FUSE_LDFLAGS)
|
||||
if(FUSE_FOUND)
|
||||
flatten_list("${FUSE_CFLAGS}" " " FUSE_CFLAGS)
|
||||
flatten_list("${FUSE_LDFLAGS}" " " FUSE_LDFLAGS)
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${FUSE_CFLAGS}")
|
||||
set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} ${FUSE_LDFLAGS}")
|
||||
MESSAGE(STATUS "Building Linux FUSE client.")
|
||||
message(STATUS "Building Linux FUSE client.")
|
||||
include_directories(${FUSE_INCLUDE_DIRS})
|
||||
ELSE(FUSE_FOUND)
|
||||
MESSAGE(STATUS "Failed to find Linux FUSE libraries or include files. Will not build FUSE client.")
|
||||
ENDIF(FUSE_FOUND)
|
||||
ELSE (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
|
||||
MESSAGE(STATUS "Non-Linux system detected. Will not build FUSE client.")
|
||||
ENDIF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
|
||||
else()
|
||||
message(STATUS "Failed to find Linux FUSE libraries or include files. Will not build FUSE client.")
|
||||
endif(FUSE_FOUND)
|
||||
else()
|
||||
message(STATUS "Non-Linux system detected. Will not build FUSE client.")
|
||||
endif()
|
||||
|
||||
IF(FUSE_FOUND)
|
||||
if(FUSE_FOUND)
|
||||
add_library(posix_util
|
||||
../util/posix_util.c
|
||||
)
|
||||
@ -88,8 +94,9 @@ IF(FUSE_FOUND)
|
||||
posix_util
|
||||
pthread
|
||||
)
|
||||
ELSE(FUSE_FOUND)
|
||||
IF(REQUIRE_FUSE)
|
||||
MESSAGE(FATAL_ERROR "Required component fuse_dfs could not be built.")
|
||||
ENDIF(REQUIRE_FUSE)
|
||||
ENDIF(FUSE_FOUND)
|
||||
set(REQUIRE_FUSE ${REQUIRE_FUSE}) # Prevent unused variable warning
|
||||
else()
|
||||
if(REQUIRE_FUSE)
|
||||
message(FATAL_ERROR "Required component fuse_dfs could not be built.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user