From 42ddb5c6fedf78ec6c183b1b66380dc6ef06ab7c Mon Sep 17 00:00:00 2001 From: Gautham B A Date: Tue, 6 Apr 2021 21:53:21 +0530 Subject: [PATCH] HDFS-15909. Make fnmatch cross platform (#2792) --- .../native/libhdfspp/lib/fs/CMakeLists.txt | 5 +- .../native/libhdfspp/lib/fs/filesystem.cc | 12 ++--- .../native/libhdfspp/lib/x-platform/syscall.h | 11 +++++ .../libhdfspp/lib/x-platform/syscall_linux.cc | 6 +++ .../lib/x-platform/syscall_windows.cc | 9 ++++ .../libhdfspp/tests/x-platform/CMakeLists.txt | 7 +++ .../tests/x-platform/syscall_common_test.cc | 47 +++++++++++++++++++ .../tests/x-platform/syscall_nix_test.cc | 35 ++++++++++++++ .../tests/x-platform/syscall_win_test.cc | 35 ++++++++++++++ 9 files changed, 159 insertions(+), 8 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_common_test.cc create mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_nix_test.cc create mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_win_test.cc diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/CMakeLists.txt index 624cda54b1..d6ea248acd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/CMakeLists.txt @@ -16,6 +16,7 @@ # limitations under the License. # -add_library(fs_obj OBJECT filesystem.cc filesystem_sync.cc filehandle.cc bad_datanode_tracker.cc namenode_operations.cc) +add_library(fs_obj OBJECT $ filesystem.cc filesystem_sync.cc filehandle.cc bad_datanode_tracker.cc namenode_operations.cc) +target_include_directories(fs_obj PRIVATE ../lib) add_dependencies(fs_obj proto) -add_library(fs $) +add_library(fs $ $) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc index ba75e86eec..741d6c783b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc @@ -25,12 +25,12 @@ #include #include #include -#include #include -#include #include +#include "x-platform/syscall.h" + #define FMT_THIS_ADDR "this=" << (void*)this namespace hdfs { @@ -722,8 +722,8 @@ void FileSystemImpl::FindShim(const Status &stat, const std::vector & for (StatInfo const& si : stat_infos) { //If we are at the last depth and it matches both path and name, we need to output it. if (operational_state->depth == shared_state->dirs.size() - 2 - && !fnmatch(shared_state->dirs[operational_state->depth + 1].c_str(), si.path.c_str(), 0) - && !fnmatch(shared_state->name.c_str(), si.path.c_str(), 0)) { + && XPlatform::Syscall::FnMatch(shared_state->dirs[operational_state->depth + 1], si.path) + && XPlatform::Syscall::FnMatch(shared_state->name, si.path)) { outputs.push_back(si); } //Skip if not directory @@ -731,7 +731,7 @@ void FileSystemImpl::FindShim(const Status &stat, const std::vector & continue; } //Checking for a match with the path at the current depth - if(!fnmatch(shared_state->dirs[operational_state->depth + 1].c_str(), si.path.c_str(), 0)){ + if(XPlatform::Syscall::FnMatch(shared_state->dirs[operational_state->depth + 1], si.path)) { //Launch a new requests for every matched directory shared_state->outstanding_requests++; auto callback = [this, si, operational_state, shared_state](const Status &stat, const std::vector & stat_infos, bool has_more) { @@ -755,7 +755,7 @@ void FileSystemImpl::FindShim(const Status &stat, const std::vector & nn_.GetListing(si.full_path, callback); } //All names that match the specified name are saved to outputs - if(!fnmatch(shared_state->name.c_str(), si.path.c_str(), 0)){ + if(XPlatform::Syscall::FnMatch(shared_state->name, si.path)) { outputs.push_back(si); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h index d162f6ff51..297acebfc5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h @@ -48,6 +48,17 @@ class Syscall { */ static int WriteToStdout(const char* message); + /** + * Checks whether the {@link str} argument matches the {@link pattern} + * argument, which is a shell wildcard pattern. + * + * @param pattern The wildcard pattern to use. + * @param str The string to match. + * @returns A boolean indicating whether the given {@link str} + * matches {@link pattern}. + */ + static bool FnMatch(const std::string& pattern, const std::string& str); + private: static bool WriteToStdoutImpl(const char* message); }; diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc index e556d9976e..2c51dbfddf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc @@ -16,6 +16,7 @@ * limitations under the License. */ +#include #include #include @@ -30,6 +31,11 @@ int XPlatform::Syscall::WriteToStdout(const char* message) { return WriteToStdoutImpl(message) ? 1 : 0; } +bool XPlatform::Syscall::FnMatch(const std::string& pattern, + const std::string& str) { + return fnmatch(pattern.c_str(), str.c_str(), 0) == 0; +} + bool XPlatform::Syscall::WriteToStdoutImpl(const char* message) { const auto message_len = strlen(message); const auto result = write(1, message, message_len); diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc index 06b0031d88..dc9ba63634 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc @@ -16,10 +16,13 @@ * limitations under the License. */ +#include #include #include "syscall.h" +#pragma comment(lib, "Shlwapi.lib") + bool XPlatform::Syscall::WriteToStdout(const std::string& message) { return WriteToStdoutImpl(message.c_str()); } @@ -28,6 +31,12 @@ int XPlatform::Syscall::WriteToStdout(const char* message) { return WriteToStdoutImpl(message) ? 1 : 0; } +bool XPlatform::Syscall::FnMatch(const std::string& pattern, + const std::string& str) { + return PathMatchSpecA(static_cast(str.c_str()), + static_cast(pattern.c_str())) == TRUE; +} + bool XPlatform::Syscall::WriteToStdoutImpl(const char* message) { auto* const stdout_handle = GetStdHandle(STD_OUTPUT_HANDLE); if (stdout_handle == INVALID_HANDLE_VALUE || stdout_handle == nullptr) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt index ac9f8fbd8c..6a7d0bec37 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt @@ -18,9 +18,16 @@ if(WIN32) add_executable(x_platform_utils_test $ utils_common_test.cc utils_test_main.cc utils_win_test.cc) + add_executable(x_platform_syscall_test $ syscall_common_test.cc utils_test_main.cc syscall_win_test.cc) else(WIN32) add_executable(x_platform_utils_test $ utils_common_test.cc utils_test_main.cc utils_nix_test.cc) + add_executable(x_platform_syscall_test $ syscall_common_test.cc utils_test_main.cc syscall_nix_test.cc) endif(WIN32) + target_include_directories(x_platform_utils_test PRIVATE ${LIBHDFSPP_LIB_DIR}) target_link_libraries(x_platform_utils_test gmock_main) add_test(x_platform_utils_test x_platform_utils_test) + +target_include_directories(x_platform_syscall_test PRIVATE ${LIBHDFSPP_LIB_DIR}) +target_link_libraries(x_platform_syscall_test gmock_main) +add_test(x_platform_syscall_test x_platform_syscall_test) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_common_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_common_test.cc new file mode 100644 index 0000000000..04da29a33f --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_common_test.cc @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +#include "x-platform/syscall.h" + +TEST(XPlatformSyscall, FnMatchBasicAsterisk) { + const std::string pattern("a*.doc"); + const std::string str("abcd.doc"); + EXPECT_TRUE(XPlatform::Syscall::FnMatch(pattern, str)); +} + +TEST(XPlatformSyscall, FnMatchBasicQuestionMark) { + const std::string pattern("a?.doc"); + const std::string str("ab.doc"); + EXPECT_TRUE(XPlatform::Syscall::FnMatch(pattern, str)); +} + +TEST(XPlatformSyscall, FnMatchNegativeAsterisk) { + const std::string pattern("a*.doc"); + const std::string str("bcd.doc"); + EXPECT_FALSE(XPlatform::Syscall::FnMatch(pattern, str)); +} + +TEST(XPlatformSyscall, FnMatchNegativeQuestionMark) { + const std::string pattern("a?.doc"); + const std::string str("abc.doc"); + EXPECT_FALSE(XPlatform::Syscall::FnMatch(pattern, str)); +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_nix_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_nix_test.cc new file mode 100644 index 0000000000..f2c753f080 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_nix_test.cc @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +#include "x-platform/syscall.h" + +TEST(XPlatformSyscall, FnMatchBasicPath) { + const std::string pattern("*.doc"); + const std::string str("some/path/abcd.doc"); + EXPECT_TRUE(XPlatform::Syscall::FnMatch(pattern, str)); +} + +TEST(XPlatformSyscall, FnMatchNegativePath) { + const std::string pattern("x*.doc"); + const std::string str("y/abcd.doc"); + EXPECT_FALSE(XPlatform::Syscall::FnMatch(pattern, str)); +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_win_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_win_test.cc new file mode 100644 index 0000000000..2d7393f4e0 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_win_test.cc @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +#include "x-platform/syscall.h" + +TEST(XPlatformSyscall, FnMatchBasicPath) { + const std::string pattern("*.doc"); + const std::string str(R"(some\path\abcd.doc)"); + EXPECT_TRUE(XPlatform::Syscall::FnMatch(pattern, str)); +} + +TEST(XPlatformSyscall, FnMatchNegativePath) { + const std::string pattern("x*.doc"); + const std::string str(R"(y\abcd.doc)"); + EXPECT_FALSE(XPlatform::Syscall::FnMatch(pattern, str)); +}