big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-16468. Define ssize_t for Windows (#4228)

Browse Source 
* Some C/C++ files use ssize_t data type.
  This isn't available for Windows and we
  need to define an alias for this and set it
  to an appropriate type to make it cross
  platform compatible.
This commit is contained in:
Gautham B A 2022-04-29 22:25:09 +05:30 committed by GitHub
parent 7bd7725532
commit 88155cebe9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 147 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/CMakeLists.txt
View File

@ -25,6 +25,7 @@ include_directories(
${CMAKE_CURRENT_SOURCE_DIR}/../libhdfs
${JNI_INCLUDE_DIRS}
${OS_DIR}
../libhdfspp/lib
)
add_library(native_mini_dfs

1
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/CMakeLists.txt
View File

@ -29,6 +29,7 @@ include_directories(
main/native
main/native/libhdfs
${OS_DIR}
../libhdfspp/lib
)
hadoop_add_dual_library(hdfs

1
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jni_helper.c
View File

@ -23,6 +23,7 @@
#include "platform.h"
#include "os/mutexes.h"
#include "os/thread_local_storage.h"
#include "x-platform/types.h"
#include <errno.h>
#include <dirent.h>

2
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/CMakeLists.txt
View File

@ -20,7 +20,7 @@
# it by add -DLIBHDFSPP_DIR=... to your cmake invocation
set(LIBHDFSPP_DIR CACHE STRING ${CMAKE_INSTALL_PREFIX})
include_directories( ${LIBHDFSPP_DIR}/include )
include_directories( ${LIBHDFSPP_DIR}/include ../../lib )
link_directories( ${LIBHDFSPP_DIR}/lib )
add_executable(cat_c cat.c)

1
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/c/cat/cat.c
View File

@ -28,6 +28,7 @@
#include "hdfspp/hdfs_ext.h"
#include "uriparser2/uriparser2.h"
#include "common/util_c.h"
#include "x-platform/types.h"
#define SCHEME "hdfs"
#define BUF_SIZE 1048576 //1 MB

2
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/examples/cc/cat/cat.cc
View File

@ -62,7 +62,6 @@ int main(int argc, char *argv[]) {
//wrapping file_raw into a unique pointer to guarantee deletion
std::unique_ptr<hdfs::FileHandle> file(file_raw);
ssize_t total_bytes_read = 0;
size_t last_bytes_read = 0;
do{
@ -71,7 +70,6 @@ int main(int argc, char *argv[]) {
if(status.ok()) {
//Writing file chunks to stdout
fwrite(input_buffer, last_bytes_read, 1, stdout);
total_bytes_read += last_bytes_read;
} else {
if(status.is_invalid_offset()){
//Reached the end of the file

1
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc
View File

@ -22,6 +22,7 @@
#include "connection/datanodeconnection.h"
#include "reader/block_reader.h"
#include "hdfspp/events.h"
#include "x-platform/types.h"
#include <future>
#include <memory>

1
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h
View File

@ -28,6 +28,7 @@
#include "bad_datanode_tracker.h"
#include "ClientNamenodeProtocol.pb.h"
#include "x-platform/types.h"
#include <memory>
#include <mutex>

1
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc
View File

@ -24,6 +24,7 @@
#include <vector>
#include "syscall.h"
#include "types.h"
bool XPlatform::Syscall::WriteToStdout(const std::string& message) {
return WriteToStdoutImpl(message.c_str());

34
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/types.h Normal file
View File

@ -0,0 +1,34 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_TYPES
#define NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_TYPES
#if _WIN64
// Windows 64-bit.
typedef long int ssize_t;
#elif _WIN32
// Windows 32-bit.
typedef int ssize_t;
#else
// ssize_t is correctly defined by taking bit-ness into account on non-Windows
// systems. So we just include the header file where ssize_t is defined.
#include <sys/types.h>
#endif
#endif

5
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt
View File

@ -31,3 +31,8 @@ add_test(x_platform_utils_test x_platform_utils_test)
target_include_directories(x_platform_syscall_test PRIVATE ${LIBHDFSPP_LIB_DIR})
target_link_libraries(x_platform_syscall_test gmock_main)
add_test(x_platform_syscall_test x_platform_syscall_test)
add_executable(x_platform_types_test types_test.cc)
target_include_directories(x_platform_types_test PRIVATE ${LIBHDFSPP_LIB_DIR})
target_link_libraries(x_platform_types_test gtest_main)
add_test(x_platform_types_test x_platform_types_test)

22
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/types_test.cc Normal file
View File

@ -0,0 +1,22 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "types_test.h"
#include "x-platform/types.h"
INSTANTIATE_TYPED_TEST_SUITE_P(SSizeTTest, XPlatformTypesTest, ssize_t);

78
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/types_test.h Normal file
View File

@ -0,0 +1,78 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef LIBHDFSPP_CROSS_PLATFORM_TYPES_TEST
#define LIBHDFSPP_CROSS_PLATFORM_TYPES_TEST
#include <limits>
#include <gtest/gtest.h>
/**
* {@class XPlatformTypesTest} tests the types defined in the XPlatform library.
*/
template <class T> class XPlatformTypesTest : public testing::Test {
public:
XPlatformTypesTest() = default;
XPlatformTypesTest(const XPlatformTypesTest &) = delete;
XPlatformTypesTest(XPlatformTypesTest &&) = delete;
XPlatformTypesTest &operator=(const XPlatformTypesTest &) = delete;
XPlatformTypesTest &operator=(XPlatformTypesTest &&) = delete;
~XPlatformTypesTest() override;
};
template <class T> XPlatformTypesTest<T>::~XPlatformTypesTest() = default;
TYPED_TEST_SUITE_P(XPlatformTypesTest);
/**
* Tests whether ssize_t can hold -1.
*/
TYPED_TEST_P(XPlatformTypesTest, SSizeTMinusOne) {
constexpr TypeParam value = -1;
ASSERT_EQ(value, -1);
}
/**
* Tests whether ssize_t can hold at least an int.
*/
TYPED_TEST_P(XPlatformTypesTest, SSizeTCanHoldInts) {
constexpr auto actual = std::numeric_limits<TypeParam>::max();
constexpr auto expected = std::numeric_limits<int>::max();
ASSERT_GE(actual, expected);
}
// For 64-bit systems.
#if _WIN64 || __x86_64__ || __ppc64__
/**
* Tests whether ssize_t can hold at least a long int.
*/
TYPED_TEST_P(XPlatformTypesTest, SSizeTCanHoldLongInts) {
constexpr auto actual = std::numeric_limits<TypeParam>::max();
constexpr auto expected = std::numeric_limits<long int>::max();
ASSERT_GE(actual, expected);
}
REGISTER_TYPED_TEST_SUITE_P(XPlatformTypesTest, SSizeTMinusOne,
SSizeTCanHoldInts, SSizeTCanHoldLongInts);
#else
REGISTER_TYPED_TEST_SUITE_P(XPlatformTypesTest, SSizeTMinusOne,
SSizeTCanHoldInts);
#endif
#endif

2
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/tools_common.cc
View File

@ -85,7 +85,6 @@ namespace hdfs {
static char input_buffer[BUF_SIZE];
void readFile(std::shared_ptr<hdfs::FileSystem> fs, std::string path, off_t offset, std::FILE* dst_file, bool to_delete) {
ssize_t total_bytes_read = 0;
size_t last_bytes_read = 0;
hdfs::FileHandle *file_raw = nullptr;
@ -103,7 +102,6 @@ namespace hdfs {
if(status.ok()) {
//Writing file chunks to stdout
fwrite(input_buffer, last_bytes_read, 1, dst_file);
total_bytes_read += last_bytes_read;
offset += last_bytes_read;
} else {
if(status.is_invalid_offset()){