big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-18129: Change URI to String in INodeLink to reduce memory footprint of ViewFileSystem

Browse Source 
Fixes #3996
This commit is contained in:
Abhishek Das 2022-02-17 20:16:19 -08:00 committed by Owen O'Malley
parent 8294bd5a37
commit da9970dd69
4 changed files with 50 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java
View File

@ -273,7 +273,7 @@ enum LinkType {
* is changed later it is then ignored (a dir with null entries)
*/
public static class INodeLink<T> extends INode<T> {
final URI[] targetDirLinkList;
final String[] targetDirLinkList;
private T targetFileSystem; // file system object created from the link.
// Function to initialize file system. Only applicable for simple links
private Function<URI, T> fileSystemInitMethod;
@ -283,7 +283,7 @@ public static class INodeLink<T> extends INode<T> {
* Construct a mergeLink or nfly.
*/
INodeLink(final String pathToNode, final UserGroupInformation aUgi,
final T targetMergeFs, final URI[] aTargetDirLinkList) {
final T targetMergeFs, final String[] aTargetDirLinkList) {
super(pathToNode, aUgi);
targetFileSystem = targetMergeFs;
targetDirLinkList = aTargetDirLinkList;
@ -294,11 +294,11 @@ public static class INodeLink<T> extends INode<T> {
*/
INodeLink(final String pathToNode, final UserGroupInformation aUgi,
Function<URI, T> createFileSystemMethod,
final URI aTargetDirLink) {
final String aTargetDirLink) throws URISyntaxException {
super(pathToNode, aUgi);
targetFileSystem = null;
targetDirLinkList = new URI[1];
targetDirLinkList[0] = aTargetDirLink;
targetDirLinkList = new String[1];
targetDirLinkList[0] = new URI(aTargetDirLink).toString();
this.fileSystemInitMethod = createFileSystemMethod;
}
@ -336,7 +336,8 @@ public T getTargetFileSystem() throws IOException {
if (targetFileSystem != null) {
return targetFileSystem;
}
targetFileSystem = fileSystemInitMethod.apply(targetDirLinkList[0]);
targetFileSystem =
fileSystemInitMethod.apply(URI.create(targetDirLinkList[0]));
if (targetFileSystem == null) {
throw new IOException(
"Could not initialize target File System for URI : " +
@ -404,7 +405,7 @@ private void createLink(final String src, final String target,
switch (linkType) {
case SINGLE:
newLink = new INodeLink<T>(fullPath, aUgi,
initAndGetTargetFs(), new URI(target));
initAndGetTargetFs(), target);
break;
case SINGLE_FALLBACK:
case MERGE_SLASH:
@ -413,10 +414,10 @@ private void createLink(final String src, final String target,
throw new IllegalArgumentException("Unexpected linkType: " + linkType);
case MERGE:
case NFLY:
final URI[] targetUris = StringUtils.stringToURI(
StringUtils.getStrings(target));
final String[] targetUris = StringUtils.getStrings(target);
newLink = new INodeLink<T>(fullPath, aUgi,
getTargetFileSystem(settings, targetUris), targetUris);
getTargetFileSystem(settings, StringUtils.stringToURI(targetUris)),
targetUris);
break;
default:
throw new IllegalArgumentException(linkType + ": Infeasible linkType");
@ -633,8 +634,7 @@ protected InodeTree(final Configuration config, final String viewName,
if (isMergeSlashConfigured) {
Preconditions.checkNotNull(mergeSlashTarget);
root = new INodeLink<T>(mountTableName, ugi,
initAndGetTargetFs(),
new URI(mergeSlashTarget));
initAndGetTargetFs(), mergeSlashTarget);
mountPoints.add(new MountPoint<T>("/", (INodeLink<T>) root));
rootFallbackLink = null;
} else {
@ -652,7 +652,7 @@ protected InodeTree(final Configuration config, final String viewName,
+ "not allowed.");
}
fallbackLink = new INodeLink<T>(mountTableName, ugi,
initAndGetTargetFs(), new URI(le.getTarget()));
initAndGetTargetFs(), le.getTarget());
continue;
case REGEX:
addRegexMountEntry(le);
@ -677,7 +677,7 @@ protected InodeTree(final Configuration config, final String viewName,
.info("Empty mount table detected for {} and considering itself "
+ "as a linkFallback.", theUri);
rootFallbackLink = new INodeLink<T>(mountTableName, ugi,
initAndGetTargetFs(), theUri);
initAndGetTargetFs(), theUri.toString());
getRootDir().addFallbackLink(rootFallbackLink);
}
}

16
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
View File

@ -216,11 +216,11 @@ public static class MountPoint {
/**
* Array of target FileSystem URIs.
*/
private final URI[] targetFileSystemURIs;
private final String[] targetFileSystemPaths;
MountPoint(Path srcPath, URI[] targetFs) {
MountPoint(Path srcPath, String[] targetFs) {
mountedOnPath = srcPath;
targetFileSystemURIs = targetFs;
targetFileSystemPaths = targetFs;
}
public Path getMountedOnPath() {
@ -228,7 +228,15 @@ public Path getMountedOnPath() {
}
public URI[] getTargetFileSystemURIs() {
return targetFileSystemURIs;
URI[] targetUris = new URI[targetFileSystemPaths.length];
for (int i = 0; i < targetFileSystemPaths.length; i++) {
targetUris[i] = URI.create(targetFileSystemPaths[i]);
}
return targetUris;
}
public String[] getTargetFileSystemPaths() {
return targetFileSystemPaths;
}
}

10
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
View File

@ -185,16 +185,18 @@ static AccessControlException readOnlyMountTable(final String operation,
static public class MountPoint {
private Path src; // the src of the mount
private URI[] targets; // target of the mount; Multiple targets imply mergeMount
MountPoint(Path srcPath, URI[] targetURIs) {
// the src of the mount
private Path src;
// Target of the mount; Multiple targets imply mergeMount
private String[] targets;
MountPoint(Path srcPath, String[] targetURIs) {
src = srcPath;
targets = targetURIs;
}
Path getSrc() {
return src;
}
URI[] getTargets() {
String[] getTargets() {
return targets;
}
}

18
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
View File

@ -1687,4 +1687,22 @@ public void testTargetFileSystemLazyInitializationForChecksumMethods()
// viewfs inner cache is disabled
assertEquals(cacheSize + 1, TestFileUtil.getCacheSize());
}
@Test
public void testInvalidMountPoints() throws Exception {
final String clusterName = "cluster" + new Random().nextInt();
Configuration config = new Configuration(conf);
config.set(ConfigUtil.getConfigViewFsPrefix(clusterName) + "." +
Constants.CONFIG_VIEWFS_LINK + "." + "/invalidPath",
"othermockfs:|mockauth/mockpath");
try {
FileSystem viewFs = FileSystem.get(
new URI("viewfs://" + clusterName + "/"), config);
fail("FileSystem should not initialize. Should fail with IOException");
} catch (IOException ex) {
assertTrue("Should get URISyntax Exception",
ex.getMessage().startsWith("URISyntax exception"));
}
}
}