big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-15909. Make fnmatch cross platform (#2792)

Browse Source
This commit is contained in:
Gautham B A 2021-04-06 21:53:21 +05:30 committed by GitHub
parent 459df41997
commit 42ddb5c6fe
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 159 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/CMakeLists.txt
View File

@ -16,6 +16,7 @@
# limitations under the License.
#
add_library(fs_obj OBJECT filesystem.cc filesystem_sync.cc filehandle.cc bad_datanode_tracker.cc namenode_operations.cc)
add_library(fs_obj OBJECT $<TARGET_OBJECTS:x_platform_obj> filesystem.cc filesystem_sync.cc filehandle.cc bad_datanode_tracker.cc namenode_operations.cc)
target_include_directories(fs_obj PRIVATE ../lib)
add_dependencies(fs_obj proto)
add_library(fs $<TARGET_OBJECTS:fs_obj>)
add_library(fs $<TARGET_OBJECTS:fs_obj> $<TARGET_OBJECTS:x_platform_obj>)

12
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
View File

@ -25,12 +25,12 @@
#include <limits>
#include <future>
#include <tuple>
#include <iostream>
#include <pwd.h>
#include <fnmatch.h>
#include <boost/asio/ip/tcp.hpp>
#include "x-platform/syscall.h"
#define FMT_THIS_ADDR "this=" << (void*)this
namespace hdfs {
@ -722,8 +722,8 @@ void FileSystemImpl::FindShim(const Status &stat, const std::vector<StatInfo> &
for (StatInfo const& si : stat_infos) {
//If we are at the last depth and it matches both path and name, we need to output it.
if (operational_state->depth == shared_state->dirs.size() - 2
&& !fnmatch(shared_state->dirs[operational_state->depth + 1].c_str(), si.path.c_str(), 0)
&& !fnmatch(shared_state->name.c_str(), si.path.c_str(), 0)) {
&& XPlatform::Syscall::FnMatch(shared_state->dirs[operational_state->depth + 1], si.path)
&& XPlatform::Syscall::FnMatch(shared_state->name, si.path)) {
outputs.push_back(si);
}
//Skip if not directory
@ -731,7 +731,7 @@ void FileSystemImpl::FindShim(const Status &stat, const std::vector<StatInfo> &
continue;
}
//Checking for a match with the path at the current depth
if(!fnmatch(shared_state->dirs[operational_state->depth + 1].c_str(), si.path.c_str(), 0)){
if(XPlatform::Syscall::FnMatch(shared_state->dirs[operational_state->depth + 1], si.path)) {
//Launch a new requests for every matched directory
shared_state->outstanding_requests++;
auto callback = [this, si, operational_state, shared_state](const Status &stat, const std::vector<StatInfo> & stat_infos, bool has_more) {
@ -755,7 +755,7 @@ void FileSystemImpl::FindShim(const Status &stat, const std::vector<StatInfo> &
nn_.GetListing(si.full_path, callback);
}
//All names that match the specified name are saved to outputs
if(!fnmatch(shared_state->name.c_str(), si.path.c_str(), 0)){
if(XPlatform::Syscall::FnMatch(shared_state->name, si.path)) {
outputs.push_back(si);
}
}

11
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h
View File

@ -48,6 +48,17 @@ class Syscall {
*/
static int WriteToStdout(const char* message);
/**
* Checks whether the {@link str} argument matches the {@link pattern}
* argument, which is a shell wildcard pattern.
*
* @param pattern The wildcard pattern to use.
* @param str The string to match.
* @returns A boolean indicating whether the given {@link str}
* matches {@link pattern}.
*/
static bool FnMatch(const std::string& pattern, const std::string& str);
private:
static bool WriteToStdoutImpl(const char* message);
};

6
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc
View File

@ -16,6 +16,7 @@
* limitations under the License.
*/
#include <fnmatch.h>
#include <unistd.h>
#include <cstring>
@ -30,6 +31,11 @@ int XPlatform::Syscall::WriteToStdout(const char* message) {
return WriteToStdoutImpl(message) ? 1 : 0;
}
bool XPlatform::Syscall::FnMatch(const std::string& pattern,
const std::string& str) {
return fnmatch(pattern.c_str(), str.c_str(), 0) == 0;
}
bool XPlatform::Syscall::WriteToStdoutImpl(const char* message) {
const auto message_len = strlen(message);
const auto result = write(1, message, message_len);

9
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc
View File

@ -16,10 +16,13 @@
* limitations under the License.
*/
#include <Shlwapi.h>
#include <Windows.h>
#include "syscall.h"
#pragma comment(lib, "Shlwapi.lib")
bool XPlatform::Syscall::WriteToStdout(const std::string& message) {
return WriteToStdoutImpl(message.c_str());
}
@ -28,6 +31,12 @@ int XPlatform::Syscall::WriteToStdout(const char* message) {
return WriteToStdoutImpl(message) ? 1 : 0;
}
bool XPlatform::Syscall::FnMatch(const std::string& pattern,
const std::string& str) {
return PathMatchSpecA(static_cast<LPCSTR>(str.c_str()),
static_cast<LPCSTR>(pattern.c_str())) == TRUE;
}
bool XPlatform::Syscall::WriteToStdoutImpl(const char* message) {
auto* const stdout_handle = GetStdHandle(STD_OUTPUT_HANDLE);
if (stdout_handle == INVALID_HANDLE_VALUE || stdout_handle == nullptr) {

7
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt
View File

@ -18,9 +18,16 @@
if(WIN32)
add_executable(x_platform_utils_test $<TARGET_OBJECTS:x_platform_obj> utils_common_test.cc utils_test_main.cc utils_win_test.cc)
add_executable(x_platform_syscall_test $<TARGET_OBJECTS:x_platform_obj> syscall_common_test.cc utils_test_main.cc syscall_win_test.cc)
else(WIN32)
add_executable(x_platform_utils_test $<TARGET_OBJECTS:x_platform_obj> utils_common_test.cc utils_test_main.cc utils_nix_test.cc)
add_executable(x_platform_syscall_test $<TARGET_OBJECTS:x_platform_obj> syscall_common_test.cc utils_test_main.cc syscall_nix_test.cc)
endif(WIN32)
target_include_directories(x_platform_utils_test PRIVATE ${LIBHDFSPP_LIB_DIR})
target_link_libraries(x_platform_utils_test gmock_main)
add_test(x_platform_utils_test x_platform_utils_test)
target_include_directories(x_platform_syscall_test PRIVATE ${LIBHDFSPP_LIB_DIR})
target_link_libraries(x_platform_syscall_test gmock_main)
add_test(x_platform_syscall_test x_platform_syscall_test)

47
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_common_test.cc Normal file
View File

@ -0,0 +1,47 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <gtest/gtest.h>
#include <string>
#include "x-platform/syscall.h"
TEST(XPlatformSyscall, FnMatchBasicAsterisk) {
const std::string pattern("a*.doc");
const std::string str("abcd.doc");
EXPECT_TRUE(XPlatform::Syscall::FnMatch(pattern, str));
}
TEST(XPlatformSyscall, FnMatchBasicQuestionMark) {
const std::string pattern("a?.doc");
const std::string str("ab.doc");
EXPECT_TRUE(XPlatform::Syscall::FnMatch(pattern, str));
}
TEST(XPlatformSyscall, FnMatchNegativeAsterisk) {
const std::string pattern("a*.doc");
const std::string str("bcd.doc");
EXPECT_FALSE(XPlatform::Syscall::FnMatch(pattern, str));
}
TEST(XPlatformSyscall, FnMatchNegativeQuestionMark) {
const std::string pattern("a?.doc");
const std::string str("abc.doc");
EXPECT_FALSE(XPlatform::Syscall::FnMatch(pattern, str));
}

35
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_nix_test.cc Normal file
View File

@ -0,0 +1,35 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <gtest/gtest.h>
#include <string>
#include "x-platform/syscall.h"
TEST(XPlatformSyscall, FnMatchBasicPath) {
const std::string pattern("*.doc");
const std::string str("some/path/abcd.doc");
EXPECT_TRUE(XPlatform::Syscall::FnMatch(pattern, str));
}
TEST(XPlatformSyscall, FnMatchNegativePath) {
const std::string pattern("x*.doc");
const std::string str("y/abcd.doc");
EXPECT_FALSE(XPlatform::Syscall::FnMatch(pattern, str));
}

35
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_win_test.cc Normal file
View File

@ -0,0 +1,35 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <gtest/gtest.h>
#include <string>
#include "x-platform/syscall.h"
TEST(XPlatformSyscall, FnMatchBasicPath) {
const std::string pattern("*.doc");
const std::string str(R"(some\path\abcd.doc)");
EXPECT_TRUE(XPlatform::Syscall::FnMatch(pattern, str));
}
TEST(XPlatformSyscall, FnMatchNegativePath) {
const std::string pattern("x*.doc");
const std::string str(R"(y\abcd.doc)");
EXPECT_FALSE(XPlatform::Syscall::FnMatch(pattern, str));
}