big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-16166. TestRawLocalFileSystemContract fails with build Docker container running on Mac.

Browse Source 
Also provided similar fix for Windows.
This commit is contained in:
Matt Foley 2019-03-05 02:40:13 -08:00
parent 1f47fb7a2f
commit f74159c8fc
1 changed files with 39 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestRawLocalFileSystemContract.java
View File

@ -18,6 +18,8 @@
package org.apache.hadoop.fs;
import java.io.File;
import java.io.IOException;
import java.util.regex.Pattern;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.test.GenericTestUtils;
@ -44,6 +46,18 @@ public class TestRawLocalFileSystemContract extends FileSystemContractBaseTest {
private final static Path TEST_BASE_DIR =
new Path(GenericTestUtils.getRandomizedTestDir().getAbsolutePath());
// These are the string values that DF sees as "Filesystem" for a
// Docker container accessing a Mac or Windows host's filesystem.
private static final String FS_TYPE_MAC = "osxfs";
private static boolean looksLikeMac(String filesys) {
return filesys.toLowerCase().contains(FS_TYPE_MAC.toLowerCase());
}
private static final Pattern HAS_DRIVE_LETTER_SPECIFIER =
Pattern.compile("^/?[a-zA-Z]:");
private static boolean looksLikeWindows(String filesys) {
return HAS_DRIVE_LETTER_SPECIFIER.matcher(filesys).find();
}
@Before
public void setUp() throws Exception {
Configuration conf = new Configuration();
@ -84,7 +98,29 @@ protected Path getTestBaseDir() {
@Override
protected boolean filesystemIsCaseSensitive() {
return !(Shell.WINDOWS || Shell.MAC);
if (Shell.WINDOWS || Shell.MAC) {
return false;
}
// osType is linux or unix-like, but it might be in a container mounting a
// Mac or Windows volume. Use DF to try to determine if this is the case.
String rfsPathStr = "uninitialized";
String rfsType;
try {
RawLocalFileSystem rfs = new RawLocalFileSystem();
Configuration conf = new Configuration();
rfs.initialize(rfs.getUri(), conf);
rfsPathStr = Path.getPathWithoutSchemeAndAuthority(
rfs.getWorkingDirectory()).toString();
File rfsPath = new File(rfsPathStr);
// DF.getFilesystem() only provides indirect info about FS type, but it's
// the best we have. `df -T` would be better, but isn't cross-platform.
rfsType = (new DF(rfsPath, conf)).getFilesystem();
LOG.info("DF.Filesystem is {} for path {}", rfsType, rfsPath);
} catch (IOException ex) {
LOG.error("DF failed on path {}", rfsPathStr);
rfsType = Shell.osType.toString();
}
return !(looksLikeMac(rfsType) || looksLikeWindows(rfsType));
}
// cross-check getPermission using both native/non-native
@ -107,8 +143,8 @@ public void testPermission() throws Exception {
// test initial permission
//
RawLocalFileSystem.DeprecatedRawLocalFileStatus fsNIO =
new RawLocalFileSystem.DeprecatedRawLocalFileStatus(
file, defaultBlockSize, rfs);
new RawLocalFileSystem.DeprecatedRawLocalFileStatus(
file, defaultBlockSize, rfs);
fsNIO.loadPermissionInfoByNativeIO();
RawLocalFileSystem.DeprecatedRawLocalFileStatus fsnonNIO =
new RawLocalFileSystem.DeprecatedRawLocalFileStatus(