From 71bbb86d69ac474596f5619d22718e9f7ff5f9dc Mon Sep 17 00:00:00 2001 From: Allen Wittenauer Date: Wed, 19 Jul 2017 17:51:14 -0700 Subject: [PATCH] HADOOP-14670. Increase minimum cmake version for all platforms Signed-off-by: Chris Douglas --- BUILDING.txt | 4 +-- dev-support/docker/Dockerfile | 17 +++++++++--- .../hadoop-common/HadoopCommon.cmake | 26 ++++++++++++------- .../hadoop-common/src/CMakeLists.txt | 2 +- .../org_apache_hadoop_io_compress_bzip2.h | 2 ++ .../src/CMakeLists.txt | 2 +- .../src/main/native/fuse-dfs/CMakeLists.txt | 2 ++ .../main/native/libhdfs-tests/CMakeLists.txt | 2 ++ .../src/main/native/libhdfs/CMakeLists.txt | 3 +++ .../src/CMakeLists.txt | 2 +- .../plugin/cmakebuilder/CompileMojo.java | 2 -- hadoop-tools/hadoop-pipes/src/CMakeLists.txt | 2 +- .../src/CMakeLists.txt | 22 ++-------------- start-build-env.sh | 2 ++ 14 files changed, 49 insertions(+), 41 deletions(-) diff --git a/BUILDING.txt b/BUILDING.txt index f9cc842a01..14deec89ea 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -7,7 +7,7 @@ Requirements: * JDK 1.8+ * Maven 3.3 or later * ProtocolBuffer 2.5.0 -* CMake 2.6 or newer (if compiling native code), must be 3.0 or newer on Mac +* CMake 3.1 or newer (if compiling native code) * Zlib devel (if compiling native code) * openssl devel (if compiling native hadoop-pipes and to get the best HDFS encryption performance) * Linux FUSE (Filesystem in Userspace) version 2.6 or above (if compiling fuse_dfs) @@ -345,7 +345,7 @@ Requirements: * JDK 1.8+ * Maven 3.0 or later * ProtocolBuffer 2.5.0 -* CMake 2.6 or newer +* CMake 3.1 or newer * Windows SDK 7.1 or Visual Studio 2010 Professional * Windows SDK 8.1 (if building CPU rate control for the container executor) * zlib headers (if building native code bindings for zlib) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 1775323ef7..31ac611f65 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -35,7 +35,6 @@ ENV DEBCONF_TERSE true RUN apt-get -q update && apt-get -q install --no-install-recommends -y \ build-essential \ bzip2 \ - cmake \ curl \ doxygen \ fuse \ @@ -89,11 +88,22 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y ant ###### RUN mkdir -p /opt/maven && \ curl -L -s -S \ - http://www-us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz \ + https://www-us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz \ -o /opt/maven.tar.gz && \ tar xzf /opt/maven.tar.gz --strip-components 1 -C /opt/maven ENV MAVEN_HOME /opt/maven -ENV PATH "$PATH:/opt/maven/bin" +ENV PATH "${PATH}:/opt/maven/bin" + +###### +# Install cmake +###### +RUN mkdir -p /opt/cmake && \ + curl -L -s -S \ + https://cmake.org/files/v3.1/cmake-3.1.0-Linux-x86_64.tar.gz \ + -o /opt/cmake.tar.gz && \ + tar xzf /opt/cmake.tar.gz --strip-components 1 -C /opt/cmake +ENV CMAKE_HOME /opt/cmake +ENV PATH "${PATH}:/opt/cmake/bin" ###### # Install findbugs @@ -104,6 +114,7 @@ RUN mkdir -p /opt/findbugs && \ -o /opt/findbugs.tar.gz && \ tar xzf /opt/findbugs.tar.gz --strip-components 1 -C /opt/findbugs ENV FINDBUGS_HOME /opt/findbugs +ENV PATH "${PATH}:/opt/findbugs/bin" #### # Install shellcheck diff --git a/hadoop-common-project/hadoop-common/HadoopCommon.cmake b/hadoop-common-project/hadoop-common/HadoopCommon.cmake index c46d9e5343..faabeedfa0 100644 --- a/hadoop-common-project/hadoop-common/HadoopCommon.cmake +++ b/hadoop-common-project/hadoop-common/HadoopCommon.cmake @@ -117,19 +117,25 @@ macro(hadoop_set_find_shared_library_without_version) endif() endmacro() -# -# Configuration. -# -# Initialise the shared gcc/g++ flags if they aren't already defined. -if(NOT DEFINED GCC_SHARED_FLAGS) - set(GCC_SHARED_FLAGS "-g -O2 -Wall -pthread -D_FILE_OFFSET_BITS=64") +# set the shared compiler flags +# support for GNU C/C++, add other compilers as necessary + +if (CMAKE_C_COMPILER_ID STREQUAL "GNU") + if(NOT DEFINED GCC_SHARED_FLAGS) + find_package(Threads REQUIRED) + if(CMAKE_USE_PTHREADS_INIT) + set(GCC_SHARED_FLAGS "-g -O2 -Wall -pthread -D_FILE_OFFSET_BITS=64") + else() + set(GCC_SHARED_FLAGS "-g -O2 -Wall -D_FILE_OFFSET_BITS=64") + endif() + endif() +elseif (CMAKE_C_COMPILER_ID STREQUAL "Clang" OR + CMAKE_C_COMPILER_ID STREQUAL "AppleClang") + set(GCC_SHARED_FLAGS "-g -O2 -Wall -D_FILE_OFFSET_BITS=64") endif() -# Add in support other compilers here, if necessary, -# the assumption is that GCC or a GCC-compatible compiler is being used. - -# Set the shared GCC-compatible compiler and linker flags. +# Set the shared linker flags. hadoop_add_compiler_flags("${GCC_SHARED_FLAGS}") hadoop_add_linker_flags("${LINKER_SHARED_FLAGS}") diff --git a/hadoop-common-project/hadoop-common/src/CMakeLists.txt b/hadoop-common-project/hadoop-common/src/CMakeLists.txt index 10b0f23ddf..b9287c0f4b 100644 --- a/hadoop-common-project/hadoop-common/src/CMakeLists.txt +++ b/hadoop-common-project/hadoop-common/src/CMakeLists.txt @@ -20,7 +20,7 @@ # CMake configuration. # -cmake_minimum_required(VERSION 2.6 FATAL_ERROR) +cmake_minimum_required(VERSION 3.1 FATAL_ERROR) list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/..) include(HadoopCommon) diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/org_apache_hadoop_io_compress_bzip2.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/org_apache_hadoop_io_compress_bzip2.h index fa525bdedb..a1db6c6f8e 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/org_apache_hadoop_io_compress_bzip2.h +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/org_apache_hadoop_io_compress_bzip2.h @@ -27,7 +27,9 @@ #include "org_apache_hadoop.h" +#ifndef HADOOP_BZIP2_LIBRARY #define HADOOP_BZIP2_LIBRARY "libbz2.so.1" +#endif /* A helper macro to convert the java 'stream-handle' to a bz_stream pointer. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt index 14b62e795b..aa690e2f41 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt @@ -16,7 +16,7 @@ # limitations under the License. # -cmake_minimum_required(VERSION 2.6 FATAL_ERROR) +cmake_minimum_required(VERSION 3.1 FATAL_ERROR) enable_testing() diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/CMakeLists.txt index 44b18e6f96..c1ba3ad9ff 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/CMakeLists.txt @@ -16,6 +16,8 @@ # limitations under the License. # +cmake_minimum_required(VERSION 3.1 FATAL_ERROR) + set(CMAKE_SKIP_RPATH TRUE) # Flatten a list into a string. diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/CMakeLists.txt index 08c9f8b2bb..08fc030bbb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/CMakeLists.txt @@ -16,6 +16,8 @@ # limitations under the License. # +cmake_minimum_required(VERSION 3.1 FATAL_ERROR) + include_directories( ${CMAKE_CURRENT_SOURCE_DIR}/../libhdfs/include ${GENERATED_JAVAH} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/CMakeLists.txt index a0cd042b93..2883585158 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/CMakeLists.txt @@ -15,6 +15,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # + +cmake_minimum_required(VERSION 3.1 FATAL_ERROR) + add_definitions(-DLIBHDFS_DLL_EXPORT) include_directories( diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt index bbeece990c..ae3b9c6029 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt @@ -16,7 +16,7 @@ # limitations under the License. # -cmake_minimum_required(VERSION 2.6 FATAL_ERROR) +cmake_minimum_required(VERSION 3.1 FATAL_ERROR) list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../../../../hadoop-common-project/hadoop-common/) include(HadoopCommon) diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/cmakebuilder/CompileMojo.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/cmakebuilder/CompileMojo.java index 0196352816..9b0bde73cd 100644 --- a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/cmakebuilder/CompileMojo.java +++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/cmakebuilder/CompileMojo.java @@ -96,8 +96,6 @@ public void execute() throws MojoExecutionException { validatePlatform(); runCMake(); runMake(); - runMake(); // The second make is a workaround for HADOOP-9215. It can be - // removed when cmake 2.6 is no longer supported. long end = System.nanoTime(); getLog().info("cmake compilation finished successfully in " + TimeUnit.MILLISECONDS.convert(end - start, TimeUnit.NANOSECONDS) + diff --git a/hadoop-tools/hadoop-pipes/src/CMakeLists.txt b/hadoop-tools/hadoop-pipes/src/CMakeLists.txt index 3b0b28c5ef..ff660bfafc 100644 --- a/hadoop-tools/hadoop-pipes/src/CMakeLists.txt +++ b/hadoop-tools/hadoop-pipes/src/CMakeLists.txt @@ -16,7 +16,7 @@ # limitations under the License. # -cmake_minimum_required(VERSION 2.6 FATAL_ERROR) +cmake_minimum_required(VERSION 3.1 FATAL_ERROR) list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../../../hadoop-common-project/hadoop-common) include(HadoopCommon) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt index 07c29bf91a..7f2b00d0e9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -cmake_minimum_required(VERSION 2.6 FATAL_ERROR) +cmake_minimum_required(VERSION 3.1 FATAL_ERROR) list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../../../../../hadoop-common-project/hadoop-common) include(HadoopCommon) @@ -29,25 +29,7 @@ set(GTEST_SRC_DIR ${CMAKE_SOURCE_DIR}/../../../../../hadoop-common-project/hadoo string(REGEX MATCH . HCD_ONE "${HADOOP_CONF_DIR}") string(COMPARE EQUAL ${HCD_ONE} / HADOOP_CONF_DIR_IS_ABS) -if (CMAKE_VERSION VERSION_LESS "3.1") - # subset of CMAKE__COMPILER_ID - # https://cmake.org/cmake/help/v3.0/variable/CMAKE_LANG_COMPILER_ID.html - if (CMAKE_C_COMPILER_ID STREQUAL "GNU" OR - CMAKE_C_COMPILER_ID STREQUAL "Clang" OR - CMAKE_C_COMPILER_ID STREQUAL "AppleClang") - set (CMAKE_C_FLAGS "-std=c99 -Wall -pedantic-errors ${CMAKE_C_FLAGS}") - elseif (CMAKE_C_COMPILER_ID STREQUAL "Intel") - set (CMAKE_C_FLAGS "-std=c99 -Wall ${CMAKE_C_FLAGS}") - elseif (CMAKE_C_COMPILER_ID STREQUAL "SunPro") - set (CMAKE_C_FLAGS "-xc99 ${CMAKE_C_FLAGS}") - endif () -else () - set (CMAKE_C_STANDARD 99) -endif () - -# Note: can't use -D_FILE_OFFSET_BITS=64, see MAPREDUCE-4258 -string(REPLACE "-D_FILE_OFFSET_BITS=64" "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") -string(REPLACE "-D_FILE_OFFSET_BITS=64" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") +set (CMAKE_C_STANDARD 99) include(CheckIncludeFiles) check_include_files("sys/types.h;sys/sysctl.h" HAVE_SYS_SYSCTL_H) diff --git a/start-build-env.sh b/start-build-env.sh index 94af7e44f2..5a18151dfa 100755 --- a/start-build-env.sh +++ b/start-build-env.sh @@ -35,7 +35,9 @@ docker build -t "hadoop-build-${USER_ID}" - < "/etc/sudoers.d/hadoop-build-${USER_ID}" ENV HOME /home/${USER_NAME} + UserSpecificDocker # By mapping the .m2 directory you can do an mvn install from