big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-15529: getChildFilesystems should include fallback fs as well (#2234). Contributed by Uma Maheswara Rao G.

Browse Source
This commit is contained in:
Uma Maheswara Rao G 2020-09-03 11:06:20 -07:00 committed by GitHub
parent 5c15815773
commit b3660d0147
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 28 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java
View File

@ -394,6 +394,15 @@ private boolean hasFallbackLink() {
return rootFallbackLink != null; return rootFallbackLink != null;
} }
/**
* @return true if the root represented as internalDir. In LinkMergeSlash,
* there will be root to root mapping. So, root does not represent as
* internalDir.
*/
protected boolean isRootInternalDir() {
return root.isInternalDir();
}
protected INodeLink<T> getRootFallbackLink() { protected INodeLink<T> getRootFallbackLink() {
Preconditions.checkState(root.isInternalDir()); Preconditions.checkState(root.isInternalDir());
return rootFallbackLink; return rootFallbackLink;

6
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
View File

@ -939,6 +939,12 @@ public FileSystem[] getChildFileSystems() {
FileSystem targetFs = mountPoint.target.targetFileSystem; FileSystem targetFs = mountPoint.target.targetFileSystem;
children.addAll(Arrays.asList(targetFs.getChildFileSystems())); children.addAll(Arrays.asList(targetFs.getChildFileSystems()));
} }
if (fsState.isRootInternalDir() && fsState.getRootFallbackLink() != null) {
children.addAll(Arrays.asList(
fsState.getRootFallbackLink().targetFileSystem
.getChildFileSystems()));
}
return children.toArray(new FileSystem[]{}); return children.toArray(new FileSystem[]{});
} }

16
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemOverloadSchemeWithHdfsScheme.java
View File

@ -476,10 +476,18 @@ public void testViewFsOverloadSchemeWithInnerCache()
// 2. Two hdfs file systems should be there if no cache. // 2. Two hdfs file systems should be there if no cache.
conf.setBoolean(Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE, false); conf.setBoolean(Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE, false);
try (FileSystem vfs = FileSystem.get(conf)) { try (FileSystem vfs = FileSystem.get(conf)) {
Assert.assertEquals(2, vfs.getChildFileSystems().length); Assert.assertEquals(isFallBackExist(conf) ? 3 : 2,
vfs.getChildFileSystems().length);
} }
} }
// HDFS-15529: if any extended tests added fallback, then getChildFileSystems
// will include fallback as well.
private boolean isFallBackExist(Configuration config) {
return config.get(ConfigUtil.getConfigViewFsPrefix(defaultFSURI
.getAuthority()) + "." + Constants.CONFIG_VIEWFS_LINK_FALLBACK) != null;
}
/** /**
* Create mount links as follows * Create mount links as follows
* hdfs://localhost:xxx/HDFSUser0 --> hdfs://localhost:xxx/HDFSUser/ * hdfs://localhost:xxx/HDFSUser0 --> hdfs://localhost:xxx/HDFSUser/
@ -501,7 +509,8 @@ public void testViewFsOverloadSchemeWithNoInnerCacheAndHdfsTargets()
conf.setBoolean(Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE, false); conf.setBoolean(Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE, false);
// Two hdfs file systems should be there if no cache. // Two hdfs file systems should be there if no cache.
try (FileSystem vfs = FileSystem.get(conf)) { try (FileSystem vfs = FileSystem.get(conf)) {
Assert.assertEquals(2, vfs.getChildFileSystems().length); Assert.assertEquals(isFallBackExist(conf) ? 3 : 2,
vfs.getChildFileSystems().length);
} }
} }
@ -528,7 +537,8 @@ public void testViewFsOverloadSchemeWithNoInnerCacheAndLocalSchemeTargets()
// cache should work. // cache should work.
conf.setBoolean(Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE, false); conf.setBoolean(Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE, false);
try (FileSystem vfs = FileSystem.get(conf)) { try (FileSystem vfs = FileSystem.get(conf)) {
Assert.assertEquals(1, vfs.getChildFileSystems().length); Assert.assertEquals(isFallBackExist(conf) ? 2 : 1,
vfs.getChildFileSystems().length);
} }
} }