From 88155cebe9461718c8d45ffacea29f15c8c4d966 Mon Sep 17 00:00:00 2001 From: Gautham B A Date: Fri, 29 Apr 2022 22:25:09 +0530 Subject: [PATCH] HDFS-16468. Define ssize_t for Windows (#4228) * Some C/C++ files use ssize_t data type. This isn't available for Windows and we need to define an alias for this and set it to an appropriate type to make it cross platform compatible. --- .../main/native/libhdfs-tests/CMakeLists.txt | 1 + .../src/main/native/libhdfs/CMakeLists.txt | 1 + .../src/main/native/libhdfs/jni_helper.c | 1 + .../libhdfspp/examples/c/cat/CMakeLists.txt | 2 +- .../native/libhdfspp/examples/c/cat/cat.c | 1 + .../native/libhdfspp/examples/cc/cat/cat.cc | 2 - .../native/libhdfspp/lib/fs/filehandle.cc | 1 + .../main/native/libhdfspp/lib/fs/filehandle.h | 1 + .../libhdfspp/lib/x-platform/syscall_linux.cc | 1 + .../native/libhdfspp/lib/x-platform/types.h | 34 ++++++++ .../libhdfspp/tests/x-platform/CMakeLists.txt | 5 ++ .../libhdfspp/tests/x-platform/types_test.cc | 22 ++++++ .../libhdfspp/tests/x-platform/types_test.h | 78 +++++++++++++++++++ .../native/libhdfspp/tools/tools_common.cc | 2 - 14 files changed, 147 insertions(+), 5 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/types.h create mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/types_test.cc create mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/types_test.h diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/CMakeLists.txt index f16cc9eb1b..44bc87d17c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/CMakeLists.txt @@ -25,6 +25,7 @@ include_directories( ${CMAKE_CURRENT_SOURCE_DIR}/../libhdfs ${JNI_INCLUDE_DIRS} ${OS_DIR} + ../libhdfspp/lib ) add_library(native_mini_dfs diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/CMakeLists.txt index a7fb311125..22d18708d6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/CMakeLists.txt @@ -29,6 +29,7 @@ include_directories( main/native main/native/libhdfs ${OS_DIR} + ../libhdfspp/lib ) hadoop_add_dual_library(hdfs diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jni_helper.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jni_helper.c index bbbc8b4602..c834c74e4c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jni_helper.c +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jni_helper.c @@ -23,6 +23,7 @@ #include "platform.h" #include "os/mutexes.h" #include "os/thread_local_storage.h" +#include "x-platform/types.h" #include #include diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/CMakeLists.txt index 41a9ee87fd..d7dfb0a744 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/CMakeLists.txt @@ -20,7 +20,7 @@ # it by add -DLIBHDFSPP_DIR=... to your cmake invocation set(LIBHDFSPP_DIR CACHE STRING ${CMAKE_INSTALL_PREFIX}) -include_directories( ${LIBHDFSPP_DIR}/include ) +include_directories( ${LIBHDFSPP_DIR}/include ../../lib ) link_directories( ${LIBHDFSPP_DIR}/lib ) add_executable(cat_c cat.c) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/cat.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/cat.c index bee5382c9e..d0de0dc5fe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/cat.c +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/cat.c @@ -28,6 +28,7 @@ #include "hdfspp/hdfs_ext.h" #include "uriparser2/uriparser2.h" #include "common/util_c.h" +#include "x-platform/types.h" #define SCHEME "hdfs" #define BUF_SIZE 1048576 //1 MB diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/cat.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/cat.cc index 9d400e7b00..584af44924 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/cat.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/cat.cc @@ -62,7 +62,6 @@ int main(int argc, char *argv[]) { //wrapping file_raw into a unique pointer to guarantee deletion std::unique_ptr file(file_raw); - ssize_t total_bytes_read = 0; size_t last_bytes_read = 0; do{ @@ -71,7 +70,6 @@ int main(int argc, char *argv[]) { if(status.ok()) { //Writing file chunks to stdout fwrite(input_buffer, last_bytes_read, 1, stdout); - total_bytes_read += last_bytes_read; } else { if(status.is_invalid_offset()){ //Reached the end of the file diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc index 7c9e24c0d8..7cfd6df3b9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc @@ -22,6 +22,7 @@ #include "connection/datanodeconnection.h" #include "reader/block_reader.h" #include "hdfspp/events.h" +#include "x-platform/types.h" #include #include diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h index 724b1a14bc..0e4eed7af4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h @@ -28,6 +28,7 @@ #include "bad_datanode_tracker.h" #include "ClientNamenodeProtocol.pb.h" +#include "x-platform/types.h" #include #include diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc index 15ec620c9a..51907634bb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc @@ -24,6 +24,7 @@ #include #include "syscall.h" +#include "types.h" bool XPlatform::Syscall::WriteToStdout(const std::string& message) { return WriteToStdoutImpl(message.c_str()); diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/types.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/types.h new file mode 100644 index 0000000000..6df5b96f39 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/types.h @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_TYPES +#define NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_TYPES + +#if _WIN64 +// Windows 64-bit. +typedef long int ssize_t; +#elif _WIN32 +// Windows 32-bit. +typedef int ssize_t; +#else +// ssize_t is correctly defined by taking bit-ness into account on non-Windows +// systems. So we just include the header file where ssize_t is defined. +#include +#endif + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt index 6a7d0bec37..e481ebc31e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt @@ -31,3 +31,8 @@ add_test(x_platform_utils_test x_platform_utils_test) target_include_directories(x_platform_syscall_test PRIVATE ${LIBHDFSPP_LIB_DIR}) target_link_libraries(x_platform_syscall_test gmock_main) add_test(x_platform_syscall_test x_platform_syscall_test) + +add_executable(x_platform_types_test types_test.cc) +target_include_directories(x_platform_types_test PRIVATE ${LIBHDFSPP_LIB_DIR}) +target_link_libraries(x_platform_types_test gtest_main) +add_test(x_platform_types_test x_platform_types_test) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/types_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/types_test.cc new file mode 100644 index 0000000000..b234fa2f10 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/types_test.cc @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "types_test.h" +#include "x-platform/types.h" + +INSTANTIATE_TYPED_TEST_SUITE_P(SSizeTTest, XPlatformTypesTest, ssize_t); diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/types_test.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/types_test.h new file mode 100644 index 0000000000..b24f657880 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/types_test.h @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_CROSS_PLATFORM_TYPES_TEST +#define LIBHDFSPP_CROSS_PLATFORM_TYPES_TEST + +#include + +#include + +/** + * {@class XPlatformTypesTest} tests the types defined in the XPlatform library. + */ +template class XPlatformTypesTest : public testing::Test { +public: + XPlatformTypesTest() = default; + XPlatformTypesTest(const XPlatformTypesTest &) = delete; + XPlatformTypesTest(XPlatformTypesTest &&) = delete; + XPlatformTypesTest &operator=(const XPlatformTypesTest &) = delete; + XPlatformTypesTest &operator=(XPlatformTypesTest &&) = delete; + ~XPlatformTypesTest() override; +}; + +template XPlatformTypesTest::~XPlatformTypesTest() = default; + +TYPED_TEST_SUITE_P(XPlatformTypesTest); + +/** + * Tests whether ssize_t can hold -1. + */ +TYPED_TEST_P(XPlatformTypesTest, SSizeTMinusOne) { + constexpr TypeParam value = -1; + ASSERT_EQ(value, -1); +} + +/** + * Tests whether ssize_t can hold at least an int. + */ +TYPED_TEST_P(XPlatformTypesTest, SSizeTCanHoldInts) { + constexpr auto actual = std::numeric_limits::max(); + constexpr auto expected = std::numeric_limits::max(); + ASSERT_GE(actual, expected); +} + +// For 64-bit systems. +#if _WIN64 || __x86_64__ || __ppc64__ +/** + * Tests whether ssize_t can hold at least a long int. + */ +TYPED_TEST_P(XPlatformTypesTest, SSizeTCanHoldLongInts) { + constexpr auto actual = std::numeric_limits::max(); + constexpr auto expected = std::numeric_limits::max(); + ASSERT_GE(actual, expected); +} + +REGISTER_TYPED_TEST_SUITE_P(XPlatformTypesTest, SSizeTMinusOne, + SSizeTCanHoldInts, SSizeTCanHoldLongInts); +#else +REGISTER_TYPED_TEST_SUITE_P(XPlatformTypesTest, SSizeTMinusOne, + SSizeTCanHoldInts); +#endif + +#endif \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cc index 6cc5a5b692..ec59dfbcd2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cc @@ -85,7 +85,6 @@ namespace hdfs { static char input_buffer[BUF_SIZE]; void readFile(std::shared_ptr fs, std::string path, off_t offset, std::FILE* dst_file, bool to_delete) { - ssize_t total_bytes_read = 0; size_t last_bytes_read = 0; hdfs::FileHandle *file_raw = nullptr; @@ -103,7 +102,6 @@ namespace hdfs { if(status.ok()) { //Writing file chunks to stdout fwrite(input_buffer, last_bytes_read, 1, dst_file); - total_bytes_read += last_bytes_read; offset += last_bytes_read; } else { if(status.is_invalid_offset()){