From 729cb3aefe71d7f728c7edea78ce7f268a1fdecb Mon Sep 17 00:00:00 2001 From: Weiwei Yang Date: Thu, 4 Jul 2019 09:37:24 +0800 Subject: [PATCH] HDFS-12748. NameNode memory leak when accessing webhdfs GETHOMEDIRECTORY. Contributed by Weiwei Yang. --- .../org/apache/hadoop/hdfs/DFSUtilClient.java | 20 +++++++++++++++++++ .../hadoop/hdfs/DistributedFileSystem.java | 8 +------- .../web/resources/NamenodeWebHdfsMethods.java | 5 ++--- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java index 77cb73831d..795ddfb379 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java @@ -994,4 +994,24 @@ public static Path makePathFromFileId(long fileId) { return new Path(sb.toString()); } + /** + * Returns current user home directory under a home directory prefix. + * The home directory prefix can be defined by + * {@link HdfsClientConfigKeys#DFS_USER_HOME_DIR_PREFIX_KEY}. + * User info is obtained from given {@link UserGroupInformation}. + * @param conf configuration + * @param ugi {@link UserGroupInformation} of current user. + * @return the home directory of current user. + */ + public static Path getHomeDirectory(Configuration conf, + UserGroupInformation ugi) { + String userHomePrefix = HdfsClientConfigKeys + .DFS_USER_HOME_DIR_PREFIX_DEFAULT; + if (conf != null) { + userHomePrefix = conf.get( + HdfsClientConfigKeys.DFS_USER_HOME_DIR_PREFIX_KEY, + HdfsClientConfigKeys.DFS_USER_HOME_DIR_PREFIX_DEFAULT); + } + return new Path(userHomePrefix + "/" + ugi.getShortUserName()); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 7956d8eda9..111b23b0ef 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -132,8 +132,6 @@ public class DistributedFileSystem extends FileSystem implements KeyProviderTokenIssuer { private Path workingDir; private URI uri; - private String homeDirPrefix = - HdfsClientConfigKeys.DFS_USER_HOME_DIR_PREFIX_DEFAULT; DFSClient dfs; private boolean verifyChecksum = true; @@ -169,9 +167,6 @@ public void initialize(URI uri, Configuration conf) throws IOException { if (host == null) { throw new IOException("Incomplete HDFS URI, no host: "+ uri); } - homeDirPrefix = conf.get( - HdfsClientConfigKeys.DFS_USER_HOME_DIR_PREFIX_KEY, - HdfsClientConfigKeys.DFS_USER_HOME_DIR_PREFIX_DEFAULT); this.dfs = new DFSClient(uri, conf, statistics); this.uri = URI.create(uri.getScheme()+"://"+uri.getAuthority()); @@ -214,8 +209,7 @@ public void setWorkingDirectory(Path dir) { @Override public Path getHomeDirectory() { - return makeQualified(new Path(homeDirPrefix + "/" - + dfs.ugi.getShortUserName())); + return makeQualified(DFSUtilClient.getHomeDirectory(getConf(), dfs.ugi)); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java index 1e8d1a5a00..ddaf0e9c6f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java @@ -1205,9 +1205,8 @@ protected Response get( return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); } case GETHOMEDIRECTORY: { - final String js = JsonUtil.toJsonString("Path", - FileSystem.get(conf != null ? conf : new Configuration()) - .getHomeDirectory().toUri().getPath()); + String userHome = DFSUtilClient.getHomeDirectory(conf, ugi).toString(); + final String js = JsonUtil.toJsonString("Path", userHome); return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); } case GETACLSTATUS: {