From bf661164077c002d9ddc9fc26ba2d89f1623ecb3 Mon Sep 17 00:00:00 2001 From: Gautham B A Date: Thu, 8 Apr 2021 22:14:47 +0530 Subject: [PATCH] HDFS-15955. Make explicit_bzero cross platform (#2875) --- .../native/libhdfspp/lib/bindings/c/hdfs.cc | 3 ++- .../native/libhdfspp/lib/x-platform/syscall.h | 12 ++++++++++ .../libhdfspp/lib/x-platform/syscall_linux.cc | 7 ++++++ .../lib/x-platform/syscall_windows.cc | 8 +++++++ .../native/libhdfspp/tests/hdfs_ext_test.cc | 3 ++- .../native/libhdfspp/tests/hdfspp_mini_dfs.h | 3 ++- .../tests/x-platform/syscall_common_test.cc | 23 +++++++++++++++++++ 7 files changed, 56 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc index efa4c75010..80f9316160 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc @@ -25,6 +25,7 @@ #include "fs/filesystem.h" #include "fs/filehandle.h" #include "x-platform/utils.h" +#include "x-platform/syscall.h" #include #include @@ -1395,7 +1396,7 @@ int hdfsGetBlockLocations(hdfsFS fs, const char *path, struct hdfsBlockLocations hdfsBlockLocations *locations = new struct hdfsBlockLocations(); (*locations_out) = locations; - explicit_bzero(locations, sizeof(*locations)); + XPlatform::Syscall::ClearBufferSafely(locations, sizeof(*locations)); locations->fileLength = ppLocations->getFileLength(); locations->isLastBlockComplete = ppLocations->isLastBlockComplete(); locations->isUnderConstruction = ppLocations->isUnderConstruction(); diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h index 297acebfc5..4f94ecbe31 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h @@ -59,6 +59,18 @@ class Syscall { */ static bool FnMatch(const std::string& pattern, const std::string& str); + /** + * Clears the given {@link buffer} upto {@link sz_bytes} by + * filling them with zeros. This method is immune to compiler + * optimizations and guarantees that the first {@link sz_bytes} of + * {@link buffer} is cleared. The {@link buffer} must be at least + * as big as {@link sz_bytes}, the behaviour is undefined otherwise. + * + * @param buffer the pointer to the buffer to clear. + * @param sz_bytes the count of the bytes to clear. + */ + static void ClearBufferSafely(void* buffer, size_t sz_bytes); + private: static bool WriteToStdoutImpl(const char* message); }; diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc index 2c51dbfddf..9821cc7110 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc @@ -41,3 +41,10 @@ bool XPlatform::Syscall::WriteToStdoutImpl(const char* message) { const auto result = write(1, message, message_len); return result == static_cast(message_len); } + +void XPlatform::Syscall::ClearBufferSafely(void* buffer, + const size_t sz_bytes) { + if (buffer != nullptr) { + explicit_bzero(buffer, sz_bytes); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc index dc9ba63634..5a3423a99f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc @@ -17,6 +17,7 @@ */ #include +#include #include #include "syscall.h" @@ -49,3 +50,10 @@ bool XPlatform::Syscall::WriteToStdoutImpl(const char* message) { WriteFile(stdout_handle, message, message_len, &bytes_written, nullptr); return result && static_cast(message_len) == bytes_written; } + +void XPlatform::Syscall::ClearBufferSafely(void* buffer, + const size_t sz_bytes) { + if (buffer != nullptr) { + SecureZeroMemory(buffer, sz_bytes); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc index fb55172633..34e53842b1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_ext_test.cc @@ -18,6 +18,7 @@ #include "hdfspp_mini_dfs.h" #include "hdfspp/hdfs_ext.h" +#include "x-platform/syscall.h" #include #include @@ -475,7 +476,7 @@ TEST_F(HdfsExtTest, TestReadStats) { hdfsFile file = hdfsOpenFile(fs, path.c_str(), O_WRONLY, 0, 0, 0); EXPECT_NE(nullptr, file); void * buf = malloc(size); - explicit_bzero(buf, size); + XPlatform::Syscall::ClearBufferSafely(buf, size); EXPECT_EQ(size, hdfsWrite(fs, file, buf, size)); free(buf); EXPECT_EQ(0, hdfsCloseFile(fs, file)); diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfspp_mini_dfs.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfspp_mini_dfs.h index 98edbdc1d6..aae8d83563 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfspp_mini_dfs.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfspp_mini_dfs.h @@ -19,6 +19,7 @@ #include "hdfs/hdfs.h" #include "hdfspp/hdfspp.h" #include +#include "x-platform/syscall.h" #include #include @@ -92,7 +93,7 @@ public: hdfsFile file = hdfsOpenFile(*this, path.c_str(), O_WRONLY, 0, 0, 0); EXPECT_NE(nullptr, file); void * buf = malloc(size); - explicit_bzero(buf, size); + XPlatform::Syscall::ClearBufferSafely(buf, size); EXPECT_EQ(1024, hdfsWrite(*this, file, buf, size)); EXPECT_EQ(0, hdfsCloseFile(*this, file)); free(buf); diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_common_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_common_test.cc index 04da29a33f..d68b2afef9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_common_test.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_common_test.cc @@ -18,7 +18,9 @@ #include +#include #include +#include #include "x-platform/syscall.h" @@ -45,3 +47,24 @@ TEST(XPlatformSyscall, FnMatchNegativeQuestionMark) { const std::string str("abc.doc"); EXPECT_FALSE(XPlatform::Syscall::FnMatch(pattern, str)); } + +TEST(XPlatformSyscall, ClearBufferSafelyChars) { + std::vector alphabets(26); + std::iota(alphabets.begin(), alphabets.end(), 'a'); + + XPlatform::Syscall::ClearBufferSafely(alphabets.data(), alphabets.size()); + for (const auto alphabet : alphabets) { + EXPECT_EQ(alphabet, '\0'); + } +} + +TEST(XPlatformSyscall, ClearBufferSafelyNumbers) { + std::vector numbers(200); + std::iota(numbers.begin(), numbers.end(), 0); + + XPlatform::Syscall::ClearBufferSafely(numbers.data(), + numbers.size() * sizeof(int)); + for (const auto number : numbers) { + EXPECT_EQ(number, 0); + } +} \ No newline at end of file