big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-15740. Add x-platform utilities (#2567)

Browse Source
This commit is contained in:
Gautham B A 2021-01-29 23:41:38 +05:30 committed by GitHub
parent fa15594ae6
commit ad483fd66e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 317 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
BUILDING.txt
View File

@ -8,10 +8,10 @@ Requirements:
* Maven 3.3 or later
* Boost 1.72 (if compiling native code)
* Protocol Buffers 3.7.1 (if compiling native code)
* CMake 3.1 or newer (if compiling native code)
* CMake 3.19 or newer (if compiling native code)
* Zlib devel (if compiling native code)
* Cyrus SASL devel (if compiling native code)
* One of the compilers that support thread_local storage: GCC 4.8.1 or later, Visual Studio,
* One of the compilers that support thread_local storage: GCC 9.3.0 or later, Visual Studio,
Clang (community version), Clang (version for iOS 9 and later) (if compiling native code)
* openssl devel (if compiling native hadoop-pipes and to get the best HDFS encryption performance)
* Linux FUSE (Filesystem in Userspace) version 2.6 or above (if compiling fuse_dfs)
@ -489,7 +489,7 @@ Requirements:
* Maven 3.0 or later
* Boost 1.72
* Protocol Buffers 3.7.1
* CMake 3.1 or newer
* CMake 3.19 or newer
* Visual Studio 2010 Professional or Higher
* Windows SDK 8.1 (if building CPU rate control for the container executor)
* zlib headers (if building native code bindings for zlib)

2
hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt
View File

@ -22,6 +22,8 @@ project(hadoop_hdfs_native_client)
enable_testing()
set(CMAKE_CXX_STANDARD 17)
list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../../../hadoop-common-project/hadoop-common)
include(HadoopCommon)

6
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
View File

@ -31,6 +31,7 @@ cmake_minimum_required(VERSION 2.8)
find_package (Boost 1.72.0 REQUIRED)
enable_testing()
set(CMAKE_CXX_STANDARD 17)
include (CTest)
SET(BUILD_SHARED_HDFSPP TRUE CACHE STRING "BUILD_SHARED_HDFSPP defaulting to 'TRUE'")
@ -78,7 +79,6 @@ add_subdirectory(${CMAKE_CURRENT_BINARY_DIR}/googletest-src
# Check if thread_local is supported
unset (THREAD_LOCAL_SUPPORTED CACHE)
set (CMAKE_CXX_STANDARD 11)
set (CMAKE_CXX_STANDARD_REQUIRED ON)
set (CMAKE_REQUIRED_LIBRARIES ${CMAKE_THREAD_LIBS_INIT})
check_cxx_source_compiles(
@ -175,13 +175,11 @@ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0")
set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0")
if(UNIX)
set (CMAKE_CXX_STANDARD 11)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic -g -fPIC -fno-strict-aliasing")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -fPIC -fno-strict-aliasing")
endif()
if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
set(CMAKE_CXX_STANDARD 11)
add_definitions(-DASIO_HAS_STD_ADDRESSOF -DASIO_HAS_STD_ARRAY -DASIO_HAS_STD_ATOMIC -DASIO_HAS_CSTDINT -DASIO_HAS_STD_SHARED_PTR -DASIO_HAS_STD_TYPE_TRAITS -DASIO_HAS_VARIADIC_TEMPLATES -DASIO_HAS_STD_FUNCTION -DASIO_HAS_STD_CHRONO -DASIO_HAS_STD_SYSTEM_ERROR)
endif ()
@ -280,7 +278,7 @@ if(NEED_LINK_DL)
endif()
set(LIBHDFSPP_VERSION "0.1.0")
set(LIBHDFSPP_ALL_OBJECTS $<TARGET_OBJECTS:bindings_c_obj> $<TARGET_OBJECTS:fs_obj> $<TARGET_OBJECTS:rpc_obj> $<TARGET_OBJECTS:reader_obj> $<TARGET_OBJECTS:proto_obj> $<TARGET_OBJECTS:connection_obj> $<TARGET_OBJECTS:common_obj> $<TARGET_OBJECTS:uriparser2_obj>)
set(LIBHDFSPP_ALL_OBJECTS $<TARGET_OBJECTS:x_platform_utils_obj> $<TARGET_OBJECTS:bindings_c_obj> $<TARGET_OBJECTS:fs_obj> $<TARGET_OBJECTS:rpc_obj> $<TARGET_OBJECTS:reader_obj> $<TARGET_OBJECTS:proto_obj> $<TARGET_OBJECTS:connection_obj> $<TARGET_OBJECTS:common_obj> $<TARGET_OBJECTS:uriparser2_obj>)
if (HADOOP_BUILD)
hadoop_add_dual_library(hdfspp ${EMPTY_FILE_CC} ${LIBHDFSPP_ALL_OBJECTS})
hadoop_target_link_dual_libraries(hdfspp

8
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CONTRIBUTING.md
View File

@ -42,14 +42,12 @@ also be followed as well as portability requirements.
Automated Formatting
--------------------
Prior to submitting a patch for code review use llvm's formatting tool, clang-format, on the .h, .c, and .cc files included in the patch. Use the -style=google switch when doing so.
Prior to submitting a patch for code review use LLVM's formatting tool, clang-format, on the .h, .c, and .cc files included in the patch. Use the -style=google switch when doing so.
Example presubmission usage:
Example pre-submission usage:
``` shell
cat my_source_file.cc | clang-format -style=goole > temp_file.cc
#optionally diff the source and temp file to get an idea what changed
mv temp_file.cc my_source_file.cc
$ clang-format -i -style=google temp_file.cc
```
* note: On some linux distributions clang-format already exists in repositories but don't show up without an appended version number. On Ubuntu you'll find it with:

1
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/CMakeLists.txt
View File

@ -16,6 +16,7 @@
# limitations under the License.
#
add_subdirectory(x-platform)
add_subdirectory(common)
add_subdirectory(fs)
add_subdirectory(reader)

6
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/CMakeLists.txt
View File

@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
add_library(bindings_c_obj OBJECT $<TARGET_OBJECTS:x_platform_utils_obj> hdfs.cc)
add_dependencies(bindings_c_obj fs rpc reader proto common fs rpc reader proto common x_platform_utils_obj)
add_library(bindings_c_obj OBJECT hdfs.cc)
add_dependencies(bindings_c_obj fs rpc reader proto common fs rpc reader proto common)
add_library(bindings_c $<TARGET_OBJECTS:bindings_c_obj>)
add_library(bindings_c $<TARGET_OBJECTS:bindings_c_obj> $<TARGET_OBJECTS:x_platform_utils_obj>)

19
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc
View File

@ -24,10 +24,9 @@
#include "common/logging.h"
#include "fs/filesystem.h"
#include "fs/filehandle.h"
#include "x-platform/utils.h"
#include <libgen.h>
#include "limits.h"
#include <limits.h>
#include <string>
#include <cstring>
#include <iostream>
@ -40,7 +39,7 @@ using namespace std::placeholders;
static constexpr tPort kDefaultPort = 8020;
/** Annotate what parts of the code below are implementatons of API functions
/** Annotate what parts of the code below are implementations of API functions
* and if they are normal vs. extended API.
*/
#define LIBHDFS_C_API
@ -767,15 +766,9 @@ void StatInfoToHdfsFileInfo(hdfsFileInfo * file_info,
LOG_WARN(kFileSystem, << "Symlink is not supported! Reporting as a file: ");
}
/* the name of the file */
char copyOfPath[PATH_MAX];
strncpy(copyOfPath, stat_info.path.c_str(), PATH_MAX);
copyOfPath[PATH_MAX - 1] = '\0'; // in case strncpy ran out of space
char * mName = basename(copyOfPath);
size_t mName_size = strlen(mName);
file_info->mName = new char[mName_size+1];
strncpy(file_info->mName, basename(copyOfPath), mName_size + 1);
const auto filename = XPlatform::Utils::Basename(stat_info.path);
file_info->mName = new char[filename.size() + 1];
strncpy(file_info->mName, filename.c_str(), filename.size() + 1);
/* the last modification time for the file in seconds */
file_info->mLastMod = (tTime) stat_info.modification_time;

19
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/CMakeLists.txt Normal file
View File

@ -0,0 +1,19 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
add_library(x_platform_utils_obj OBJECT utils.cc)

41
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.cc Normal file
View File

@ -0,0 +1,41 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "x-platform/utils.h"
#include <filesystem>
#include <string>
#include <vector>
std::string XPlatform::Utils::Basename(const std::string& file_path) {
if (file_path.empty()) {
return ".";
}
const std::filesystem::path path(file_path);
std::vector<std::string> parts;
for (const auto& part : std::filesystem::path(file_path)) {
parts.emplace_back(part.string());
}
/* Handle the case of trailing slash */
if (parts.back().empty()) {
parts.pop_back();
}
return parts.back();
}

44
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.h Normal file
View File

@ -0,0 +1,44 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_UTILS
#define NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_UTILS
#include <string>
/**
* The {@link XPlatform} namespace contains components that
* aid in writing cross-platform code.
*/
namespace XPlatform {
class Utils {
public:
/**
* A cross-platform implementation of basename in linux.
* Please refer https://www.man7.org/linux/man-pages/man3/basename.3.html
* for more details.
*
* @param file_path The input path to get the basename.
*
* @returns The trailing component of the given {@link file_path}
*/
static std::string Basename(const std::string& file_path);
};
} // namespace XPlatform
#endif

11
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt
View File

@ -65,6 +65,8 @@ endfunction(add_memcheck_test)
#
#
add_subdirectory(x-platform)
add_executable(uri_test uri_test.cc)
target_link_libraries(uri_test common gmock_main ${CMAKE_THREAD_LIBS_INIT})
add_memcheck_test(uri uri_test)
@ -110,7 +112,7 @@ add_executable(hdfs_builder_test hdfs_builder_test.cc)
target_link_libraries(hdfs_builder_test test_common gmock_main bindings_c fs rpc proto common reader connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT})
add_memcheck_test(hdfs_builder_test hdfs_builder_test)
add_executable(logging_test logging_test.cc)
add_executable(logging_test logging_test.cc $<TARGET_OBJECTS:x_platform_utils_obj>)
target_link_libraries(logging_test common gmock_main bindings_c fs rpc proto common reader connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT})
add_memcheck_test(logging_test logging_test)
@ -142,11 +144,10 @@ include_directories (
${CMAKE_CURRENT_SOURCE_DIR}/../../libhdfs-tests/
)
add_library(hdfspp_test_shim_static STATIC hdfs_shim.c libhdfs_wrapper.c libhdfspp_wrapper.cc ${LIBHDFSPP_BINDING_C}/hdfs.cc)
add_library(hdfspp_test_static STATIC ${LIBHDFSPP_BINDING_C}/hdfs.cc)
# Add dependencies
add_library(hdfspp_test_shim_static STATIC $<TARGET_OBJECTS:x_platform_utils_obj> hdfs_shim.c libhdfs_wrapper.c libhdfspp_wrapper.cc ${LIBHDFSPP_BINDING_C}/hdfs.cc)
add_dependencies(hdfspp_test_shim_static proto)
add_library(hdfspp_test_static STATIC $<TARGET_OBJECTS:x_platform_utils_obj> ${LIBHDFSPP_BINDING_C}/hdfs.cc)
add_dependencies(hdfspp_test_static proto)
# TODO: get all of the mini dfs library bits here in one place

26
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt Normal file
View File

@ -0,0 +1,26 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
if(WIN32)
add_executable(x_platform_utils_test $<TARGET_OBJECTS:x_platform_utils_obj> utils_common_test.cc utils_test_main.cc utils_win_test.cc)
else(WIN32)
add_executable(x_platform_utils_test $<TARGET_OBJECTS:x_platform_utils_obj> utils_common_test.cc utils_test_main.cc utils_nix_test.cc)
endif(WIN32)
target_include_directories(x_platform_utils_test PRIVATE ${LIBHDFSPP_LIB_DIR})
target_link_libraries(x_platform_utils_test gmock_main)
add_test(x_platform_utils_test x_platform_utils_test)

45
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/utils_common_test.cc Normal file
View File

@ -0,0 +1,45 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <gtest/gtest.h>
#include <string>
#include "x-platform/utils.h"
TEST(XPlatformUtils, BasenameEmpty) {
const std::string expected(".");
const auto actual = XPlatform::Utils::Basename("");
EXPECT_EQ(expected, actual);
}
TEST(XPlatformUtils, BasenameRelativePath) {
const std::string expected("x");
const auto actual = XPlatform::Utils::Basename("x");
EXPECT_EQ(expected, actual);
}
TEST(XPlatformUtils, BasenameSpecialFiles) {
const std::string current_dir_expected(".");
const auto current_dir_actual = XPlatform::Utils::Basename(".");
EXPECT_EQ(current_dir_expected, current_dir_actual);
const std::string parent_dir_expected("..");
const auto parent_dir_actual = XPlatform::Utils::Basename("..");
EXPECT_EQ(parent_dir_expected, parent_dir_actual);
}

43
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/utils_nix_test.cc Normal file
View File

@ -0,0 +1,43 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <gtest/gtest.h>
#include <string>
#include "x-platform/utils.h"
TEST(XPlatformUtils, BasenameRoot) {
const std::string nix_expected("/");
const auto nix_actual = XPlatform::Utils::Basename("/");
EXPECT_EQ(nix_expected, nix_actual);
}
TEST(XPlatformUtils, BasenameTrailingSlash) {
const std::string expected("def");
const std::string nix_path("/abc/def/");
const auto nix_actual = XPlatform::Utils::Basename(nix_path);
EXPECT_EQ(expected, nix_actual);
}
TEST(XPlatformUtils, BasenameBasic) {
const std::string expected("def");
const std::string nix_path("/abc/def");
const auto nix_actual = XPlatform::Utils::Basename(nix_path);
EXPECT_EQ(expected, nix_actual);
}

24
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/utils_test_main.cc Normal file
View File

@ -0,0 +1,24 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <gtest/gtest.h>
int main(int argc, char* argv[]) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

49
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/utils_win_test.cc Normal file
View File

@ -0,0 +1,49 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <gtest/gtest.h>
#include <string>
#include "x-platform/utils.h"
TEST(XPlatformUtils, BasenameRoot) {
const std::string win_expected(R"(\)");
const auto win_actual = XPlatform::Utils::Basename(R"(\)");
EXPECT_EQ(win_expected, win_actual);
}
TEST(XPlatformUtils, BasenameRootLabel) {
const std::string win_expected(R"(C:\)");
const auto win_actual = XPlatform::Utils::Basename(R"(C:\)");
EXPECT_EQ(win_expected, win_actual);
}
TEST(XPlatformUtils, BasenameTrailingSlash) {
const std::string expected("def");
const std::string win_path(R"(C:\abc\def\)");
const auto win_actual = XPlatform::Utils::Basename(win_path);
EXPECT_EQ(expected, win_actual);
}
TEST(XPlatformUtils, BasenameBasic) {
const std::string expected("def");
const std::string win_path(R"(C:\abc\def)");
const auto win_actual = XPlatform::Utils::Basename(win_path);
EXPECT_EQ(expected, win_actual);
}