HDFS-15740. Add x-platform utilities (#2567)

BUILDING.txt

@@ -8,10 +8,10 @@ Requirements:
 * Maven 3.3 or later
 * Boost 1.72 (if compiling native code)
 * Protocol Buffers 3.7.1 (if compiling native code)
-* CMake 3.1 or newer (if compiling native code)
+* CMake 3.19 or newer (if compiling native code)
 * Zlib devel (if compiling native code)
 * Cyrus SASL devel (if compiling native code)
-* One of the compilers that support thread_local storage: GCC 4.8.1 or later, Visual Studio,
+* One of the compilers that support thread_local storage: GCC 9.3.0 or later, Visual Studio,
   Clang (community version), Clang (version for iOS 9 and later) (if compiling native code)
 * openssl devel (if compiling native hadoop-pipes and to get the best HDFS encryption performance)
 * Linux FUSE (Filesystem in Userspace) version 2.6 or above (if compiling fuse_dfs)
@@ -489,7 +489,7 @@ Requirements:
 * Maven 3.0 or later
 * Boost 1.72
 * Protocol Buffers 3.7.1
-* CMake 3.1 or newer
+* CMake 3.19 or newer
 * Visual Studio 2010 Professional or Higher
 * Windows SDK 8.1 (if building CPU rate control for the container executor)
 * zlib headers (if building native code bindings for zlib)

hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt

@@ -22,6 +22,8 @@ project(hadoop_hdfs_native_client)
 
 enable_testing()
 
+set(CMAKE_CXX_STANDARD 17)
+
 list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../../../hadoop-common-project/hadoop-common)
 include(HadoopCommon)
 

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt

@@ -31,6 +31,7 @@ cmake_minimum_required(VERSION 2.8)
 find_package (Boost 1.72.0 REQUIRED)
 
 enable_testing()
+set(CMAKE_CXX_STANDARD 17)
 include (CTest)
 
 SET(BUILD_SHARED_HDFSPP TRUE CACHE STRING "BUILD_SHARED_HDFSPP defaulting to 'TRUE'")
@@ -78,7 +79,6 @@ add_subdirectory(${CMAKE_CURRENT_BINARY_DIR}/googletest-src
 
 # Check if thread_local is supported
 unset (THREAD_LOCAL_SUPPORTED CACHE)
-set (CMAKE_CXX_STANDARD 11)
 set (CMAKE_CXX_STANDARD_REQUIRED ON)
 set (CMAKE_REQUIRED_LIBRARIES ${CMAKE_THREAD_LIBS_INIT})
 check_cxx_source_compiles(
@@ -175,13 +175,11 @@ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0")
 set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0")
 
 if(UNIX)
-set (CMAKE_CXX_STANDARD 11)
 set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic -g -fPIC -fno-strict-aliasing")
 set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -fPIC -fno-strict-aliasing")
 endif()
 
 if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
-    set(CMAKE_CXX_STANDARD 11)
     add_definitions(-DASIO_HAS_STD_ADDRESSOF -DASIO_HAS_STD_ARRAY -DASIO_HAS_STD_ATOMIC -DASIO_HAS_CSTDINT -DASIO_HAS_STD_SHARED_PTR -DASIO_HAS_STD_TYPE_TRAITS -DASIO_HAS_VARIADIC_TEMPLATES -DASIO_HAS_STD_FUNCTION -DASIO_HAS_STD_CHRONO -DASIO_HAS_STD_SYSTEM_ERROR)
 endif ()
 
@@ -280,7 +278,7 @@ if(NEED_LINK_DL)
 endif()
 
 set(LIBHDFSPP_VERSION "0.1.0")
-set(LIBHDFSPP_ALL_OBJECTS $<TARGET_OBJECTS:bindings_c_obj> $<TARGET_OBJECTS:fs_obj> $<TARGET_OBJECTS:rpc_obj> $<TARGET_OBJECTS:reader_obj> $<TARGET_OBJECTS:proto_obj> $<TARGET_OBJECTS:connection_obj> $<TARGET_OBJECTS:common_obj> $<TARGET_OBJECTS:uriparser2_obj>)
+set(LIBHDFSPP_ALL_OBJECTS $<TARGET_OBJECTS:x_platform_utils_obj> $<TARGET_OBJECTS:bindings_c_obj> $<TARGET_OBJECTS:fs_obj> $<TARGET_OBJECTS:rpc_obj> $<TARGET_OBJECTS:reader_obj> $<TARGET_OBJECTS:proto_obj> $<TARGET_OBJECTS:connection_obj> $<TARGET_OBJECTS:common_obj> $<TARGET_OBJECTS:uriparser2_obj>)
 if (HADOOP_BUILD)
   hadoop_add_dual_library(hdfspp ${EMPTY_FILE_CC} ${LIBHDFSPP_ALL_OBJECTS})
   hadoop_target_link_dual_libraries(hdfspp

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CONTRIBUTING.md

@@ -42,14 +42,12 @@ also be followed as well as portability requirements.
 Automated Formatting
 --------------------
 
-Prior to submitting a patch for code review use llvm's formatting tool, clang-format, on the .h, .c, and .cc files included in the patch.  Use the -style=google switch when doing so.
+Prior to submitting a patch for code review use LLVM's formatting tool, clang-format, on the .h, .c, and .cc files included in the patch.  Use the -style=google switch when doing so.
 
-Example presubmission usage:
+Example pre-submission usage:
 
 ``` shell
-cat my_source_file.cc | clang-format -style=goole > temp_file.cc
+$ clang-format -i -style=google temp_file.cc
-#optionally diff the source and temp file to get an idea what changed
-mv temp_file.cc my_source_file.cc
 ```
 
 * note: On some linux distributions clang-format already exists in repositories but don't show up without an appended version number.  On Ubuntu you'll find it with:

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/CMakeLists.txt

@@ -16,6 +16,7 @@
 # limitations under the License.
 #
 
+add_subdirectory(x-platform)
 add_subdirectory(common)
 add_subdirectory(fs)
 add_subdirectory(reader)

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/CMakeLists.txt

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+add_library(bindings_c_obj OBJECT $<TARGET_OBJECTS:x_platform_utils_obj> hdfs.cc)
+add_dependencies(bindings_c_obj fs rpc reader proto common fs rpc reader proto common x_platform_utils_obj)
 
-add_library(bindings_c_obj OBJECT hdfs.cc)
+add_library(bindings_c $<TARGET_OBJECTS:bindings_c_obj> $<TARGET_OBJECTS:x_platform_utils_obj>)
-add_dependencies(bindings_c_obj fs rpc reader proto common fs rpc reader proto common)
-add_library(bindings_c $<TARGET_OBJECTS:bindings_c_obj>)

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc

@@ -24,10 +24,9 @@
 #include "common/logging.h"
 #include "fs/filesystem.h"
 #include "fs/filehandle.h"
+#include "x-platform/utils.h"
 
-
+#include <limits.h>
-#include <libgen.h>
-#include "limits.h"
 #include <string>
 #include <cstring>
 #include <iostream>
@@ -40,7 +39,7 @@ using namespace std::placeholders;
 
 static constexpr tPort kDefaultPort = 8020;
 
-/** Annotate what parts of the code below are implementatons of API functions
+/** Annotate what parts of the code below are implementations of API functions
  *  and if they are normal vs. extended API.
  */
 #define LIBHDFS_C_API
@@ -767,15 +766,9 @@ void StatInfoToHdfsFileInfo(hdfsFileInfo * file_info,
     LOG_WARN(kFileSystem, << "Symlink is not supported! Reporting as a file: ");
   }
 
-  /* the name of the file */
+  const auto filename = XPlatform::Utils::Basename(stat_info.path);
-  char copyOfPath[PATH_MAX];
+  file_info->mName = new char[filename.size() + 1];
-  strncpy(copyOfPath, stat_info.path.c_str(), PATH_MAX);
+  strncpy(file_info->mName, filename.c_str(), filename.size() + 1);
-  copyOfPath[PATH_MAX - 1] = '\0'; // in case strncpy ran out of space
-
-  char * mName = basename(copyOfPath);
-  size_t mName_size = strlen(mName);
-  file_info->mName = new char[mName_size+1];
-  strncpy(file_info->mName, basename(copyOfPath), mName_size + 1);
 
   /* the last modification time for the file in seconds */
   file_info->mLastMod = (tTime) stat_info.modification_time;

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/CMakeLists.txt

@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+add_library(x_platform_utils_obj OBJECT utils.cc)

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.cc

@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "x-platform/utils.h"
+
+#include <filesystem>
+#include <string>
+#include <vector>
+
+std::string XPlatform::Utils::Basename(const std::string& file_path) {
+  if (file_path.empty()) {
+    return ".";
+  }
+
+  const std::filesystem::path path(file_path);
+  std::vector<std::string> parts;
+  for (const auto& part : std::filesystem::path(file_path)) {
+    parts.emplace_back(part.string());
+  }
+
+  /* Handle the case of trailing slash */
+  if (parts.back().empty()) {
+    parts.pop_back();
+  }
+  return parts.back();
+}

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.h

@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_UTILS
+#define NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_UTILS
+
+#include <string>
+
+/**
+ * The {@link XPlatform} namespace contains components that
+ * aid in writing cross-platform code.
+ */
+namespace XPlatform {
+class Utils {
+ public:
+  /**
+   * A cross-platform implementation of basename in linux.
+   * Please refer https://www.man7.org/linux/man-pages/man3/basename.3.html
+   * for more details.
+   *
+   * @param file_path The input path to get the basename.
+   *
+   * @returns The trailing component of the given {@link file_path}
+   */
+  static std::string Basename(const std::string& file_path);
+};
+}  // namespace XPlatform
+
+#endif

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt

@@ -65,6 +65,8 @@ endfunction(add_memcheck_test)
 #
 #
 
+add_subdirectory(x-platform)
+
 add_executable(uri_test uri_test.cc)
 target_link_libraries(uri_test common gmock_main ${CMAKE_THREAD_LIBS_INIT})
 add_memcheck_test(uri uri_test)
@@ -110,7 +112,7 @@ add_executable(hdfs_builder_test hdfs_builder_test.cc)
 target_link_libraries(hdfs_builder_test test_common gmock_main bindings_c fs rpc proto common reader connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT})
 add_memcheck_test(hdfs_builder_test hdfs_builder_test)
 
-add_executable(logging_test logging_test.cc)
+add_executable(logging_test logging_test.cc $<TARGET_OBJECTS:x_platform_utils_obj>)
 target_link_libraries(logging_test common gmock_main bindings_c fs rpc proto common reader connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT})
 add_memcheck_test(logging_test logging_test)
 
@@ -142,11 +144,10 @@ include_directories (
     ${CMAKE_CURRENT_SOURCE_DIR}/../../libhdfs-tests/
 )
 
-add_library(hdfspp_test_shim_static STATIC hdfs_shim.c libhdfs_wrapper.c libhdfspp_wrapper.cc ${LIBHDFSPP_BINDING_C}/hdfs.cc)
+add_library(hdfspp_test_shim_static STATIC $<TARGET_OBJECTS:x_platform_utils_obj> hdfs_shim.c libhdfs_wrapper.c libhdfspp_wrapper.cc ${LIBHDFSPP_BINDING_C}/hdfs.cc)
-add_library(hdfspp_test_static STATIC ${LIBHDFSPP_BINDING_C}/hdfs.cc)
-
-# Add dependencies
 add_dependencies(hdfspp_test_shim_static proto)
+
+add_library(hdfspp_test_static STATIC $<TARGET_OBJECTS:x_platform_utils_obj> ${LIBHDFSPP_BINDING_C}/hdfs.cc)
 add_dependencies(hdfspp_test_static proto)
 
 # TODO: get all of the mini dfs library bits here in one place

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt

@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+if(WIN32)
+    add_executable(x_platform_utils_test $<TARGET_OBJECTS:x_platform_utils_obj> utils_common_test.cc utils_test_main.cc utils_win_test.cc)
+else(WIN32)
+    add_executable(x_platform_utils_test $<TARGET_OBJECTS:x_platform_utils_obj> utils_common_test.cc utils_test_main.cc utils_nix_test.cc)
+endif(WIN32)
+target_include_directories(x_platform_utils_test PRIVATE ${LIBHDFSPP_LIB_DIR})
+target_link_libraries(x_platform_utils_test gmock_main)
+add_test(x_platform_utils_test x_platform_utils_test)

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/utils_common_test.cc

@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <gtest/gtest.h>
+
+#include <string>
+
+#include "x-platform/utils.h"
+
+TEST(XPlatformUtils, BasenameEmpty) {
+  const std::string expected(".");
+  const auto actual = XPlatform::Utils::Basename("");
+  EXPECT_EQ(expected, actual);
+}
+
+TEST(XPlatformUtils, BasenameRelativePath) {
+  const std::string expected("x");
+  const auto actual = XPlatform::Utils::Basename("x");
+  EXPECT_EQ(expected, actual);
+}
+
+TEST(XPlatformUtils, BasenameSpecialFiles) {
+  const std::string current_dir_expected(".");
+  const auto current_dir_actual = XPlatform::Utils::Basename(".");
+  EXPECT_EQ(current_dir_expected, current_dir_actual);
+
+  const std::string parent_dir_expected("..");
+  const auto parent_dir_actual = XPlatform::Utils::Basename("..");
+  EXPECT_EQ(parent_dir_expected, parent_dir_actual);
+}

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/utils_nix_test.cc

@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <gtest/gtest.h>
+
+#include <string>
+
+#include "x-platform/utils.h"
+
+TEST(XPlatformUtils, BasenameRoot) {
+  const std::string nix_expected("/");
+  const auto nix_actual = XPlatform::Utils::Basename("/");
+  EXPECT_EQ(nix_expected, nix_actual);
+}
+
+TEST(XPlatformUtils, BasenameTrailingSlash) {
+  const std::string expected("def");
+  const std::string nix_path("/abc/def/");
+  const auto nix_actual = XPlatform::Utils::Basename(nix_path);
+  EXPECT_EQ(expected, nix_actual);
+}
+
+TEST(XPlatformUtils, BasenameBasic) {
+  const std::string expected("def");
+  const std::string nix_path("/abc/def");
+  const auto nix_actual = XPlatform::Utils::Basename(nix_path);
+  EXPECT_EQ(expected, nix_actual);
+}

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/utils_test_main.cc

@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <gtest/gtest.h>
+
+int main(int argc, char* argv[]) {
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/utils_win_test.cc

@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <gtest/gtest.h>
+
+#include <string>
+
+#include "x-platform/utils.h"
+
+TEST(XPlatformUtils, BasenameRoot) {
+  const std::string win_expected(R"(\)");
+  const auto win_actual = XPlatform::Utils::Basename(R"(\)");
+  EXPECT_EQ(win_expected, win_actual);
+}
+
+TEST(XPlatformUtils, BasenameRootLabel) {
+  const std::string win_expected(R"(C:\)");
+  const auto win_actual = XPlatform::Utils::Basename(R"(C:\)");
+  EXPECT_EQ(win_expected, win_actual);
+}
+
+TEST(XPlatformUtils, BasenameTrailingSlash) {
+  const std::string expected("def");
+  const std::string win_path(R"(C:\abc\def\)");
+  const auto win_actual = XPlatform::Utils::Basename(win_path);
+  EXPECT_EQ(expected, win_actual);
+}
+
+TEST(XPlatformUtils, BasenameBasic) {
+  const std::string expected("def");
+  const std::string win_path(R"(C:\abc\def)");
+  const auto win_actual = XPlatform::Utils::Basename(win_path);
+  EXPECT_EQ(expected, win_actual);
+}