big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-12026: libhdfs++: Fix compilation errors and warnings when compiling with Clang. Contributed by Anatoli Shein.

Browse Source
This commit is contained in:
James Clampffer 2017-07-25 12:25:45 -04:00
parent 53e40bc718
commit 033433bce7
20 changed files with 242 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dev-support/docker/Dockerfile
View File

@ -38,10 +38,12 @@ ENV DEBCONF_TERSE true
# WARNING: DO NOT PUT JAVA APPS HERE! Otherwise they will install default
# Ubuntu Java. See Java section below!
######
<<<<<<< 5f556eacf0fedbaec3cdb819847c7c6c3bacc048
RUN apt-get -q update && apt-get -q install -y \
apt-utils \
build-essential \
bzip2 \
clang \
curl \
doxygen \
fuse \

72
hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
View File

@ -247,5 +247,77 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
</plugins>
</build>
</profile>
<profile>
<id>test-patch</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<properties>
<runningWithNative>true</runningWithNative>
</properties>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
<id>make_altern</id>
<phase>compile</phase>
<goals><goal>run</goal></goals>
<configuration>
<target>
<mkdir dir="${project.build.directory}/altern"/>
<condition property="c_compiler" value="clang" else="gcc">
<contains string="${env.CC}" substring="gcc"/>
</condition>
<condition property="cxx_compiler" value="clang++" else="g++">
<contains string="${env.CXX}" substring="g++"/>
</condition>
<exec executable="cmake" dir="${project.build.directory}/altern" failonerror="true">
<arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/altern/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DHADOOP_BUILD=1 -DREQUIRE_LIBWEBHDFS=${require.libwebhdfs} -DREQUIRE_FUSE=${require.fuse} -DREQUIRE_VALGRIND=${require.valgrind} "/>
<arg line="-DCMAKE_C_COMPILER=${c_compiler} -DCMAKE_CXX_COMPILER=${cxx_compiler}"/>
<arg line="${native_cmake_args}"/>
</exec>
<exec executable="make" dir="${project.build.directory}/altern" failonerror="true">
<arg line="${native_make_args}"/>
</exec>
</target>
</configuration>
</execution>
<execution>
<id>native_tests_altern</id>
<phase>test</phase>
<goals><goal>run</goal></goals>
<configuration>
<skip>${skipTests}</skip>
<target>
<property name="compile_classpath" refid="maven.compile.classpath"/>
<property name="test_classpath" refid="maven.test.classpath"/>
<exec executable="ctest" failonerror="true" dir="${project.build.directory}/altern">
<arg line="--output-on-failure"/>
<arg line="${native_ctest_args}"/>
<env key="CLASSPATH" value="${test_classpath}:${compile_classpath}"/>
<!-- Make sure libhadoop.so is on LD_LIBRARY_PATH. -->
<env key="LD_LIBRARY_PATH" value="${env.LD_LIBRARY_PATH}:${project.build.directory}/altern/target/usr/local/lib:${hadoop.common.build.dir}/native/target/usr/local/lib"/>
</exec>
</target>
</configuration>
</execution>
<execution>
<id>clean_altern</id>
<phase>test</phase>
<goals><goal>run</goal></goals>
<configuration>
<target>
<delete dir="${project.build.directory}/altern" includeemptydirs="true"/>
</target>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>

46
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
View File

@ -42,6 +42,47 @@ find_package(CyrusSASL)
find_package(GSasl)
find_package(Threads)
include(CheckCXXSourceCompiles)
# Check if thread_local is supported
unset (THREAD_LOCAL_SUPPORTED CACHE)
set (CMAKE_REQUIRED_DEFINITIONS "-std=c++11")
set (CMAKE_REQUIRED_LIBRARIES ${CMAKE_THREAD_LIBS_INIT})
check_cxx_source_compiles(
"#include <thread>
int main(void) {
thread_local int s;
return 0;
}"
THREAD_LOCAL_SUPPORTED)
if (NOT THREAD_LOCAL_SUPPORTED)
message(FATAL_ERROR
"FATAL ERROR: The required feature thread_local storage is not supported by your compiler. \
Known compilers that support this feature: GCC, Visual Studio, Clang (community version), \
Clang (version for iOS 9 and later).")
endif (NOT THREAD_LOCAL_SUPPORTED)
# Check if PROTOC library was compiled with the compatible compiler by trying
# to compile some dummy code
unset (PROTOC_IS_COMPATIBLE CACHE)
set (CMAKE_REQUIRED_LIBRARIES ${PROTOBUF_LIBRARY} ${PROTOBUF_PROTOC_LIBRARY})
check_cxx_source_compiles(
"#include <google/protobuf/io/printer.h>
#include <string>
int main(void) {
::google::protobuf::io::ZeroCopyOutputStream *out = NULL;
::google::protobuf::io::Printer printer(out, '$');
printer.PrintRaw(std::string(\"test\"));
return 0;
}"
PROTOC_IS_COMPATIBLE)
if (NOT PROTOC_IS_COMPATIBLE)
message(FATAL_ERROR
"FATAL ERROR: the Protocol Buffers Library and the Libhdfs++ Library must both be compiled \
with the same (or compatible) compiler. Normally only the same major versions of the same \
compiler are compatible with each other.")
endif (NOT PROTOC_IS_COMPATIBLE)
find_program(MEMORYCHECK_COMMAND valgrind HINTS ${VALGRIND_DIR} )
set(MEMORYCHECK_COMMAND_OPTIONS "--trace-children=yes --leak-check=full --error-exitcode=1")
message(STATUS "valgrind location: ${MEMORYCHECK_COMMAND}")
@ -104,6 +145,11 @@ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic -std=c++11 -g -
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -fPIC -fno-strict-aliasing")
endif()
if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
add_definitions(-DASIO_HAS_STD_ADDRESSOF -DASIO_HAS_STD_ARRAY -DASIO_HAS_STD_ATOMIC -DASIO_HAS_CSTDINT -DASIO_HAS_STD_SHARED_PTR -DASIO_HAS_STD_TYPE_TRAITS -DASIO_HAS_VARIADIC_TEMPLATES -DASIO_HAS_STD_FUNCTION -DASIO_HAS_STD_CHRONO -DASIO_HAS_STD_SYSTEM_ERROR)
endif ()
# Mac OS 10.7 and later deprecates most of the methods in OpenSSL.
# Add -Wno-deprecated-declarations to avoid the warnings.
if(APPLE)

92
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/block_location.h
View File

@ -23,152 +23,152 @@ namespace hdfs {
class DNInfo {
public:
DNInfo() : xfer_port(-1), info_port(-1), IPC_port(-1), info_secure_port(-1) {}
DNInfo() : xfer_port_(-1), info_port_(-1), IPC_port_(-1), info_secure_port_(-1) {}
std::string getHostname() const {
return hostname;
return hostname_;
}
void setHostname(const std::string & hostname) {
this->hostname = hostname;
this->hostname_ = hostname;
}
std::string getIPAddr() const {
return ip_addr;
return ip_addr_;
}
void setIPAddr(const std::string & ip_addr) {
this->ip_addr = ip_addr;
this->ip_addr_ = ip_addr;
}
std::string getNetworkLocation() const {
return network_location;
return network_location_;
}
void setNetworkLocation(const std::string & location) {
this->network_location = location;
this->network_location_ = location;
}
int getXferPort() const {
return xfer_port;
return xfer_port_;
}
void setXferPort(int xfer_port) {
this->xfer_port = xfer_port;
this->xfer_port_ = xfer_port;
}
int getInfoPort() const {
return info_port;
return info_port_;
}
void setInfoPort(int info_port) {
this->info_port = info_port;
this->info_port_ = info_port;
}
int getIPCPort() const {
return IPC_port;
return IPC_port_;
}
void setIPCPort(int IPC_port) {
this->IPC_port = IPC_port;
this->IPC_port_ = IPC_port;
}
int getInfoSecurePort() const {
return info_secure_port;
return info_secure_port_;
}
void setInfoSecurePort(int info_secure_port) {
this->info_secure_port = info_secure_port;
this->info_secure_port_ = info_secure_port;
}
private:
std::string hostname;
std::string ip_addr;
std::string network_location;
int xfer_port;
int info_port;
int IPC_port;
int info_secure_port;
std::string hostname_;
std::string ip_addr_;
std::string network_location_;
int xfer_port_;
int info_port_;
int IPC_port_;
int info_secure_port_;
};
class BlockLocation {
public:
bool isCorrupt() const {
return corrupt;
return corrupt_;
}
void setCorrupt(bool corrupt) {
this->corrupt = corrupt;
this->corrupt_ = corrupt;
}
int64_t getLength() const {
return length;
return length_;
}
void setLength(int64_t length) {
this->length = length;
this->length_ = length;
}
int64_t getOffset() const {
return offset;
return offset_;
}
void setOffset(int64_t offset) {
this->offset = offset;
this->offset_ = offset;
}
const std::vector<DNInfo> & getDataNodes() const {
return dn_info;
return dn_info_;
}
void setDataNodes(const std::vector<DNInfo> & dn_info) {
this->dn_info = dn_info;
this->dn_info_ = dn_info;
}
private:
bool corrupt;
int64_t length;
int64_t offset; // Offset of the block in the file
std::vector<DNInfo> dn_info; // Info about who stores each block
bool corrupt_;
int64_t length_;
int64_t offset_; // Offset of the block in the file
std::vector<DNInfo> dn_info_; // Info about who stores each block
};
class FileBlockLocation {
public:
uint64_t getFileLength() {
return fileLength;
return fileLength_;
}
void setFileLength(uint64_t fileLength) {
this->fileLength = fileLength;
this->fileLength_ = fileLength;
}
bool isLastBlockComplete() const {
return this->lastBlockComplete;
return this->lastBlockComplete_;
}
void setLastBlockComplete(bool lastBlockComplete) {
this->lastBlockComplete = lastBlockComplete;
this->lastBlockComplete_ = lastBlockComplete;
}
bool isUnderConstruction() const {
return underConstruction;
return underConstruction_;
}
void setUnderConstruction(bool underConstruction) {
this->underConstruction = underConstruction;
this->underConstruction_ = underConstruction;
}
const std::vector<BlockLocation> & getBlockLocations() const {
return blockLocations;
return blockLocations_;
}
void setBlockLocations(const std::vector<BlockLocation> & blockLocations) {
this->blockLocations = blockLocations;
this->blockLocations_ = blockLocations;
}
private:
uint64_t fileLength;
bool lastBlockComplete;
bool underConstruction;
std::vector<BlockLocation> blockLocations;
uint64_t fileLength_;
bool lastBlockComplete_;
bool underConstruction_;
std::vector<BlockLocation> blockLocations_;
};
} // namespace hdfs

12
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h
View File

@ -93,7 +93,7 @@ class IoService : public std::enable_shared_from_this<IoService>
**/
class NodeExclusionRule {
public:
virtual ~NodeExclusionRule(){};
virtual ~NodeExclusionRule();
virtual bool IsBadNode(const std::string &node_uuid) = 0;
};
@ -258,7 +258,7 @@ class FileSystem {
* this method call. The blocks will be populated or removed in the
* background as the result of the routine block maintenance procedures.
*
* @param src file name
* @param path file name
* @param replication new replication
*/
virtual void SetReplication(const std::string & path, int16_t replication, std::function<void(const Status &)> handler) = 0;
@ -266,7 +266,7 @@ class FileSystem {
/**
* Sets the modification and access time of the file to the specified time.
* @param src The string representation of the path
* @param path The string representation of the path
* @param mtime The number of milliseconds since Jan 1, 1970.
* Setting mtime to -1 means that modification time should not
* be set by this call.
@ -297,7 +297,7 @@ class FileSystem {
* Retrieves the file system information as a whole, such as the total raw size of all files in the filesystem
* and the raw capacity of the filesystem
*
* @param FsInfo struct to be populated by GetFsStats
* FsInfo struct is populated by GetFsStats
**/
virtual void GetFsStats(
const std::function<void(const Status &, const FsInfo &)> &handler) = 0;
@ -357,7 +357,6 @@ class FileSystem {
* Rename - Rename file.
* @param oldPath The path of the source file. (must be non-empty)
* @param newPath The path of the destination file. (must be non-empty)
* @return Returns 0 on success, -1 on error.
*/
virtual void Rename(const std::string &oldPath, const std::string &newPath,
const std::function<void(const Status &)> &handler) = 0;
@ -379,7 +378,6 @@ class FileSystem {
* @param path file path
* @param username If it is empty, the original username remains unchanged.
* @param groupname If it is empty, the original groupname remains unchanged.
* @param recursive If true, the change will be propagated recursively.
*/
virtual void SetOwner(const std::string & path, const std::string & username,
const std::string & groupname, const std::function<void(const Status &)> &handler) = 0;
@ -469,7 +467,7 @@ class FileSystem {
* Note that it is an error to destroy the filesystem from within a filesystem
* callback. It will lead to a deadlock and the termination of the process.
*/
virtual ~FileSystem() {};
virtual ~FileSystem();
/**

6
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h
View File

@ -28,9 +28,9 @@ namespace hdfs {
struct NamenodeInfo {
NamenodeInfo(const std::string &nameservice, const std::string &nodename, const URI &uri) :
nameservice(nameservice), name(nodename), uri(uri) {}
NamenodeInfo(){};
NamenodeInfo(const std::string &nameservice_, const std::string &nodename_, const URI &uri_) :
nameservice(nameservice_), name(nodename_), uri(uri_) {}
NamenodeInfo(){}
//nameservice this belongs to
std::string nameservice;
//node name

2
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h
View File

@ -26,7 +26,7 @@ namespace hdfs {
class Status {
public:
// Create a success status.
Status() : code_(0) {};
Status() : code_(0) {}
// Note: Avoid calling the Status constructors directly, call the factory methods instead

2
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/auth_info.h
View File

@ -19,7 +19,7 @@
#ifndef LIB_FS_AUTHINFO_H
#define LIB_FS_AUTHINFO_H
#include <optional.hpp>
#include "common/optional_wrapper.h"
namespace hdfs {

14
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/configuration.h
View File

@ -27,7 +27,7 @@
#include <set>
#include <istream>
#include <stdint.h>
#include <optional.hpp>
#include "common/optional_wrapper.h"
namespace hdfs {
@ -76,8 +76,8 @@ protected:
struct ConfigData {
std::string value;
bool final;
ConfigData() : final(false){};
ConfigData(const std::string &value) : value(value), final(false) {}
ConfigData() : final(false){}
ConfigData(const std::string &value_) : value(value_), final(false) {}
void operator=(const std::string &new_value) {
value = new_value;
final = false;
@ -85,9 +85,9 @@ protected:
};
typedef std::map<std::string, ConfigData> ConfigMap;
Configuration() {};
Configuration(ConfigMap &src_map) : raw_values_(src_map){};
Configuration(const ConfigMap &src_map) : raw_values_(src_map){};
Configuration() {}
Configuration(ConfigMap &src_map) : raw_values_(src_map){}
Configuration(const ConfigMap &src_map) : raw_values_(src_map){}
static std::vector<std::string> GetDefaultFilenames();
@ -98,7 +98,7 @@ protected:
static std::string fixCase(const std::string &in) {
std::string result(in);
for (auto & c: result) c = (char) toupper(c);
for (auto & c: result) c = static_cast<char>(toupper(c));
return result;
}
};

3
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/libhdfs_events_impl.h
View File

@ -20,9 +20,8 @@
#define LIBHDFSPP_COMMON_LIBHDFSEVENTS_IMPL
#include "hdfspp/events.h"
#include "common/optional_wrapper.h"
#include <optional.hpp>
#include <functional>
namespace hdfs {

43
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/optional_wrapper.h Normal file
View File

@ -0,0 +1,43 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef COMMON_OPTIONAL_WRAPPER_H_
#define COMMON_OPTIONAL_WRAPPER_H_
#ifdef __clang__
#pragma clang diagnostic push
#if __has_warning("-Wweak-vtables")
#pragma clang diagnostic ignored "-Wweak-vtables"
#endif
#if __has_warning("-Wreserved-id-macro")
#pragma clang diagnostic ignored "-Wreserved-id-macro"
#endif
#if __has_warning("-Wextra-semi")
#pragma clang diagnostic ignored "-Wextra-semi"
#endif
#define TR2_OPTIONAL_DISABLE_EMULATION_OF_TYPE_TRAITS //For Clang < 3_4_2
#endif
#include <optional.hpp>
#ifdef __clang__
#undef TR2_OPTIONAL_DISABLE_EMULATION_OF_TYPE_TRAITS //For Clang < 3_4_2
#pragma clang diagnostic pop
#endif
#endif //COMMON_OPTIONAL_WRAPPER_H_

2
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.h
View File

@ -160,7 +160,7 @@ class SwappableCallbackHolder {
} else if (callback_accessed_) {
// Common case where callback has been invoked but caller may not know
LOG_DEBUG(kAsyncRuntime, << "AtomicSwapCallback called after callback has been accessed");
return false;
return callback_;
}
CallbackType old = callback_;

2
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/bad_datanode_tracker.cc
View File

@ -20,6 +20,8 @@
namespace hdfs {
NodeExclusionRule::~NodeExclusionRule() {}
BadDataNodeTracker::BadDataNodeTracker(const Options& options)
: timeout_duration_(options.host_exclusion_duration),
test_clock_shift_(0) {}

2
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h
View File

@ -36,7 +36,7 @@
namespace hdfs {
class BlockReader;
class BlockReaderOptions;
struct BlockReaderOptions;
class DataNodeConnection;
/*

2
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
View File

@ -70,6 +70,8 @@ Status FileSystem::CheckValidReplication(uint16_t replication) {
return Status::OK();
}
FileSystem::~FileSystem() {}
/*****************************************************************************
* FILESYSTEM BASE CLASS
****************************************************************************/

2
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc
View File

@ -20,7 +20,7 @@
#include "common/util.h"
#include "common/logging.h"
#include "common/namenode_info.h"
#include "optional.hpp"
#include "common/optional_wrapper.h"
#include <algorithm>

2
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_engine.h
View File

@ -20,7 +20,7 @@
#define LIB_RPC_SASLENGINE_H
#include "hdfspp/status.h"
#include "optional.hpp"
#include "common/optional_wrapper.h"
#include <vector>

3
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc
View File

@ -19,6 +19,7 @@
#include "rpc_engine.h"
#include "rpc_connection.h"
#include "common/logging.h"
#include "common/optional_wrapper.h"
#include "sasl_engine.h"
#include "sasl_protocol.h"
@ -33,8 +34,6 @@
#endif
#endif
#include <optional.hpp>
namespace hdfs {
using namespace hadoop::common;

4
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ioservice_test.cc
View File

@ -99,5 +99,7 @@ int main(int argc, char *argv[]) {
// The following line must be executed to initialize Google Mock
// (and Google Test) before running the tests.
::testing::InitGoogleMock(&argc, argv);
return RUN_ALL_TESTS();
int exit_code = RUN_ALL_TESTS();
google::protobuf::ShutdownProtobufLibrary();
return exit_code;
}

4
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/node_exclusion_test.cc
View File

@ -80,5 +80,7 @@ int main(int argc, char *argv[]) {
// The following line must be executed to initialize Google Mock
// (and Google Test) before running the tests.
::testing::InitGoogleMock(&argc, argv);
return RUN_ALL_TESTS();
int exit_code = RUN_ALL_TESTS();
google::protobuf::ShutdownProtobufLibrary();
return exit_code;
}