From 29df73dc0572cf8e450568f1d0301e8d55d64b48 Mon Sep 17 00:00:00 2001 From: Colin Patrick Mccabe Date: Tue, 30 Jun 2015 15:54:38 -0700 Subject: [PATCH] HDFS-8635. Migrate HDFS native build to new CMake framework (Alan Burlison via Colin P. McCabe) --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hadoop-hdfs/src/CMakeLists.txt | 133 ++++++------------ .../src/contrib/libwebhdfs/CMakeLists.txt | 8 +- .../libwebhdfs/resources/FindJansson.cmake | 9 +- .../src/main/native/fuse-dfs/CMakeLists.txt | 91 ++++++------ 5 files changed, 102 insertions(+), 142 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index b065f98e73..65f286d7da 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -691,6 +691,9 @@ Release 2.8.0 - UNRELEASED HDFS-8493. Consolidate truncate() related implementation in a single class. (Rakesh R via wheat9) + HDFS-8635. Migrate HDFS native build to new CMake framework (Alan Burlison + via Colin P. McCabe) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt index 563727b44f..b4a3b40c10 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt @@ -18,106 +18,59 @@ cmake_minimum_required(VERSION 2.6 FATAL_ERROR) -# Default to release builds -set(CMAKE_BUILD_TYPE, Release) +list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../../../hadoop-common-project/hadoop-common) +include(HadoopCommon) -include(../../../hadoop-common-project/hadoop-common/src/JNIFlags.cmake NO_POLICY_SCOPE) +# +# Main configuration +# -# Compile a library with both shared and static variants -function(add_dual_library LIBNAME) - add_library(${LIBNAME} SHARED ${ARGN}) - add_library(${LIBNAME}_static STATIC ${ARGN}) - # Linux builds traditionally ship a libhdfs.a (static linking) and libhdfs.so - # (dynamic linking). On Windows, we cannot use the same base name for both - # static and dynamic, because Windows does not use distinct file extensions - # for a statically linked library vs. a DLL import library. Both use the - # .lib extension. On Windows, we'll build the static library as - # hdfs_static.lib. - if (NOT WIN32) - set_target_properties(${LIBNAME}_static PROPERTIES OUTPUT_NAME ${LIBNAME}) - endif (NOT WIN32) -endfunction(add_dual_library) - -# Link both a static and a dynamic target against some libraries -function(target_link_dual_libraries LIBNAME) - target_link_libraries(${LIBNAME} ${ARGN}) - target_link_libraries(${LIBNAME}_static ${ARGN}) -endfunction(target_link_dual_libraries) - -function(output_directory TGT DIR) - SET_TARGET_PROPERTIES(${TGT} PROPERTIES - RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") - SET_TARGET_PROPERTIES(${TGT} PROPERTIES - ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") - SET_TARGET_PROPERTIES(${TGT} PROPERTIES - LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") -endfunction(output_directory TGT DIR) - -function(dual_output_directory TGT DIR) - output_directory(${TGT} "${DIR}") - output_directory(${TGT}_static "${DIR}") -endfunction(dual_output_directory TGT DIR) - -# Flatten a list into a string. -function(FLATTEN_LIST INPUT SEPARATOR OUTPUT) - string (REPLACE ";" "${SEPARATOR}" _TMPS "${INPUT}") - set (${OUTPUT} "${_TMPS}" PARENT_SCOPE) -endfunction() +# The caller must specify where the generated headers have been placed. +if(NOT GENERATED_JAVAH) + message(FATAL_ERROR "You must set the CMake variable GENERATED_JAVAH") +endif() # Check to see if our compiler and linker support the __thread attribute. -# On Linux and some other operating systems, this is a more efficient -# alternative to POSIX thread local storage. -INCLUDE(CheckCSourceCompiles) -CHECK_C_SOURCE_COMPILES("int main(void) { static __thread int i = 0; return 0; }" HAVE_BETTER_TLS) +# On Linux and some other operating systems, this is a more efficient +# alternative to POSIX thread local storage. +include(CheckCSourceCompiles) +check_c_source_compiles("int main(void) { static __thread int i = 0; return 0; }" HAVE_BETTER_TLS) # Check to see if we have Intel SSE intrinsics. -CHECK_C_SOURCE_COMPILES("#include \nint main(void) { __m128d sum0 = _mm_set_pd(0.0,0.0); return 0; }" HAVE_INTEL_SSE_INTRINSICS) +check_c_source_compileS("#include \nint main(void) { __m128d sum0 = _mm_set_pd(0.0,0.0); return 0; }" HAVE_INTEL_SSE_INTRINSICS) # Check if we need to link dl library to get dlopen. # dlopen on Linux is in separate library but on FreeBSD its in libc -INCLUDE(CheckLibraryExists) -CHECK_LIBRARY_EXISTS(dl dlopen "" NEED_LINK_DL) +include(CheckLibraryExists) +check_library_exists(dl dlopen "" NEED_LINK_DL) -find_package(JNI REQUIRED) -if (NOT GENERATED_JAVAH) - # Must identify where the generated headers have been placed - MESSAGE(FATAL_ERROR "You must set the CMake variable GENERATED_JAVAH") -endif (NOT GENERATED_JAVAH) - -if (WIN32) +if(WIN32) + # Set the optimizer level. set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /O2") - # Set warning level 4. set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /W4") - # Skip "unreferenced formal parameter". set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4100") - # Skip "conditional expression is constant". set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4127") - # Skip deprecated POSIX function warnings. set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_CRT_NONSTDC_NO_DEPRECATE") - # Skip CRT non-secure function warnings. If we can convert usage of # strerror, getenv and ctime to their secure CRT equivalents, then we can # re-enable the CRT non-secure function warnings. set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_CRT_SECURE_NO_WARNINGS") - # Omit unneeded headers. set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DWIN32_LEAN_AND_MEAN") - set(OS_DIR main/native/libhdfs/os/windows) set(OUT_DIR target/bin) -else (WIN32) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wall -O2") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT -D_GNU_SOURCE") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64") +else() set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fvisibility=hidden") set(OS_DIR main/native/libhdfs/os/posix) - set(OS_LINK_LIBRARIES pthread) set(OUT_DIR target/usr/local/lib) -endif (WIN32) +endif() + +# Configure JNI. +include(HadoopJNI) add_definitions(-DLIBHDFS_DLL_EXPORT) @@ -132,9 +85,9 @@ include_directories( ) set(_FUSE_DFS_VERSION 0.1.0) -CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h) +configure_file(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h) -add_dual_library(hdfs +hadoop_add_dual_library(hdfs main/native/libhdfs/exception.c main/native/libhdfs/jni_helper.c main/native/libhdfs/hdfs.c @@ -142,17 +95,17 @@ add_dual_library(hdfs ${OS_DIR}/mutexes.c ${OS_DIR}/thread_local_storage.c ) -if (NEED_LINK_DL) +if(NEED_LINK_DL) set(LIB_DL dl) -endif(NEED_LINK_DL) +endif() -target_link_dual_libraries(hdfs +hadoop_target_link_dual_libraries(hdfs ${JAVA_JVM_LIBRARY} ${LIB_DL} ${OS_LINK_LIBRARIES} ) -dual_output_directory(hdfs ${OUT_DIR}) +hadoop_dual_output_directory(hdfs ${OUT_DIR}) set(LIBHDFS_VERSION "0.0.0") set_target_properties(hdfs PROPERTIES SOVERSION ${LIBHDFS_VERSION}) @@ -232,25 +185,21 @@ target_link_libraries(test_htable # Skip vecsum on Windows. This could be made to work in the future by # introducing an abstraction layer over the sys/mman.h functions. -if (NOT WIN32) - add_executable(test_libhdfs_vecsum - main/native/libhdfs/test/vecsum.c - ) - if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") +if(NOT WIN32) + add_executable(test_libhdfs_vecsum main/native/libhdfs/test/vecsum.c) + if(CMAKE_SYSTEM_NAME MATCHES "Darwin") + target_link_libraries(test_libhdfs_vecsum + hdfs + pthread) + else() target_link_libraries(test_libhdfs_vecsum hdfs pthread - ) - else (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") - target_link_libraries(test_libhdfs_vecsum - hdfs - pthread - rt - ) - endif (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") -endif(NOT WIN32) + rt) + endif() +endif() -IF(REQUIRE_LIBWEBHDFS) +if(REQUIRE_LIBWEBHDFS) add_subdirectory(contrib/libwebhdfs) -ENDIF(REQUIRE_LIBWEBHDFS) +endif() add_subdirectory(main/native/fuse-dfs) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/CMakeLists.txt index 44f0c42baa..dc74feb7ed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/CMakeLists.txt @@ -19,12 +19,12 @@ find_package(CURL REQUIRED) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} -"${CMAKE_SOURCE_DIR}/contrib/libwebhdfs/resources/") + "${CMAKE_SOURCE_DIR}/contrib/libwebhdfs/resources/") find_package(Jansson REQUIRED) include_directories(${JANSSON_INCLUDE_DIR}) -add_dual_library(webhdfs +hadoop_add_dual_library(webhdfs src/hdfs_web.c src/hdfs_http_client.c src/hdfs_http_query.c @@ -35,13 +35,13 @@ add_dual_library(webhdfs ../../${OS_DIR}/thread_local_storage.c ../../main/native/libhdfs/common/htable.c ) -target_link_dual_libraries(webhdfs +hadoop_target_link_dual_libraries(webhdfs ${JAVA_JVM_LIBRARY} ${CURL_LIBRARY} ${JANSSON_LIBRARY} pthread ) -dual_output_directory(webhdfs target) +hadoop_dual_output_directory(webhdfs target) set(LIBWEBHDFS_VERSION "0.0.0") set_target_properties(webhdfs PROPERTIES SOVERSION ${LIBWEBHDFS_VERSION}) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/FindJansson.cmake b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/FindJansson.cmake index cb8c8dbb57..b8c67eae32 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/FindJansson.cmake +++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/FindJansson.cmake @@ -5,9 +5,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -24,7 +24,8 @@ # JANSSON_DEFINITIONS - Compiler switches required for using Jansson find_path(JANSSON_INCLUDE_DIR jansson.h - /usr/incluce + /usr/include + /usr/include/jansson /usr/local/include ) find_library(JANSSON_LIBRARY NAMES jansson @@ -39,4 +40,4 @@ include(FindPackageHandleStandardArgs) find_package_handle_standard_args(Jansson DEFAULT_MSG JANSSON_LIBRARY JANSSON_INCLUDE_DIR) -mark_as_advanced(JANSSON_INCLUDE_DIR JANSSON_LIBRARY ) \ No newline at end of file +mark_as_advanced(JANSSON_INCLUDE_DIR JANSSON_LIBRARY ) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt index 8a8ea5b2b3..7a5e5a1a77 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt @@ -18,57 +18,63 @@ set(CMAKE_SKIP_RPATH TRUE) +# Flatten a list into a string. +function(flatten_list INPUT SEPARATOR OUTPUT) + string (REPLACE ";" "${SEPARATOR}" _TMPS "${INPUT}") + set (${OUTPUT} "${_TMPS}" PARENT_SCOPE) +endfunction() + # Find Linux FUSE -IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") +if(${CMAKE_SYSTEM_NAME} MATCHES "Linux") find_package(PkgConfig REQUIRED) pkg_check_modules(FUSE fuse) - IF(FUSE_FOUND) - FLATTEN_LIST("${FUSE_CFLAGS}" " " FUSE_CFLAGS) - FLATTEN_LIST("${FUSE_LDFLAGS}" " " FUSE_LDFLAGS) + if(FUSE_FOUND) + flatten_list("${FUSE_CFLAGS}" " " FUSE_CFLAGS) + flatten_list("${FUSE_LDFLAGS}" " " FUSE_LDFLAGS) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${FUSE_CFLAGS}") set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} ${FUSE_LDFLAGS}") - MESSAGE(STATUS "Building Linux FUSE client.") + message(STATUS "Building Linux FUSE client.") include_directories(${FUSE_INCLUDE_DIRS}) - ELSE(FUSE_FOUND) - MESSAGE(STATUS "Failed to find Linux FUSE libraries or include files. Will not build FUSE client.") - ENDIF(FUSE_FOUND) -ELSE (${CMAKE_SYSTEM_NAME} MATCHES "Linux") - MESSAGE(STATUS "Non-Linux system detected. Will not build FUSE client.") -ENDIF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") + else() + message(STATUS "Failed to find Linux FUSE libraries or include files. Will not build FUSE client.") + endif(FUSE_FOUND) +else() + message(STATUS "Non-Linux system detected. Will not build FUSE client.") +endif() -IF(FUSE_FOUND) +if(FUSE_FOUND) add_library(posix_util ../util/posix_util.c ) add_executable(fuse_dfs fuse_dfs.c - fuse_options.c - fuse_connect.c - fuse_impls_access.c - fuse_impls_chmod.c - fuse_impls_chown.c - fuse_impls_create.c - fuse_impls_flush.c - fuse_impls_getattr.c - fuse_impls_mkdir.c - fuse_impls_mknod.c - fuse_impls_open.c - fuse_impls_read.c - fuse_impls_readdir.c - fuse_impls_release.c - fuse_impls_rename.c - fuse_impls_rmdir.c - fuse_impls_statfs.c - fuse_impls_symlink.c - fuse_impls_truncate.c - fuse_impls_unlink.c - fuse_impls_utimens.c + fuse_options.c + fuse_connect.c + fuse_impls_access.c + fuse_impls_chmod.c + fuse_impls_chown.c + fuse_impls_create.c + fuse_impls_flush.c + fuse_impls_getattr.c + fuse_impls_mkdir.c + fuse_impls_mknod.c + fuse_impls_open.c + fuse_impls_read.c + fuse_impls_readdir.c + fuse_impls_release.c + fuse_impls_rename.c + fuse_impls_rmdir.c + fuse_impls_statfs.c + fuse_impls_symlink.c + fuse_impls_truncate.c + fuse_impls_unlink.c + fuse_impls_utimens.c fuse_impls_write.c - fuse_init.c - fuse_stat_struct.c - fuse_trash.c - fuse_users.c + fuse_init.c + fuse_stat_struct.c + fuse_trash.c + fuse_users.c ) target_link_libraries(fuse_dfs ${FUSE_LIBRARIES} @@ -88,8 +94,9 @@ IF(FUSE_FOUND) posix_util pthread ) -ELSE(FUSE_FOUND) - IF(REQUIRE_FUSE) - MESSAGE(FATAL_ERROR "Required component fuse_dfs could not be built.") - ENDIF(REQUIRE_FUSE) -ENDIF(FUSE_FOUND) + set(REQUIRE_FUSE ${REQUIRE_FUSE}) # Prevent unused variable warning +else() + if(REQUIRE_FUSE) + message(FATAL_ERROR "Required component fuse_dfs could not be built.") + endif() +endif()