From 91d0b4727066241c900037f9a10e28b143489172 Mon Sep 17 00:00:00 2001 From: Jing Zhao Date: Sat, 7 Dec 2013 21:10:53 +0000 Subject: [PATCH] HDFS-4983. Numeric usernames do not work with WebHDFS FS. Contributed by Yongjun Zhang. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1548968 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 ++ .../server/namenode/NameNodeHttpServer.java | 6 +++- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 2 ++ .../hadoop/hdfs/web/resources/UserParam.java | 25 +++++++++++++---- .../src/main/resources/hdfs-default.xml | 8 ++++++ .../apache/hadoop/hdfs/web/TestWebHDFS.java | 28 +++++++++++++++++++ .../hadoop/hdfs/web/resources/TestParam.java | 15 ++++++++++ 8 files changed, 83 insertions(+), 6 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index e13e80e4a8..daa80388c9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -584,6 +584,9 @@ Release 2.4.0 - UNRELEASED HDFS-5633. Improve OfflineImageViewer to use less memory. (jing9) + HDFS-4983. Numeric usernames do not work with WebHDFS FS. (Yongjun Zhang via + jing9) + OPTIMIZATIONS HDFS-5239. Allow FSNamesystem lock fairness to be configurable (daryn) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 3dc06ba8a3..6caef9389c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -164,6 +164,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final int DFS_NAMENODE_REPLICATION_STREAMS_HARD_LIMIT_DEFAULT = 4; public static final String DFS_WEBHDFS_ENABLED_KEY = "dfs.webhdfs.enabled"; public static final boolean DFS_WEBHDFS_ENABLED_DEFAULT = true; + public static final String DFS_WEBHDFS_USER_PATTERN_KEY = "dfs.webhdfs.user.provider.user.pattern"; + public static final String DFS_WEBHDFS_USER_PATTERN_DEFAULT = "^[A-Za-z_][A-Za-z0-9._-]*[$]?$"; public static final String DFS_PERMISSIONS_ENABLED_KEY = "dfs.permissions.enabled"; public static final boolean DFS_PERMISSIONS_ENABLED_DEFAULT = true; public static final String DFS_PERMISSIONS_SUPERUSERGROUP_KEY = "dfs.permissions.superusergroup"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java index b72d546d14..a9d89dd545 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hdfs.web.AuthFilter; import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; import org.apache.hadoop.hdfs.web.resources.Param; +import org.apache.hadoop.hdfs.web.resources.UserParam; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.http.HttpServer; import org.apache.hadoop.net.NetUtils; @@ -73,7 +74,10 @@ public class NameNodeHttpServer { private void initWebHdfs(Configuration conf) throws IOException { if (WebHdfsFileSystem.isEnabled(conf, HttpServer.LOG)) { - //add SPNEGO authentication filter for webhdfs + // set user pattern based on configuration file + UserParam.setUserPattern(conf.get(DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY, DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT)); + + // add SPNEGO authentication filter for webhdfs final String name = "SPNEGO"; final String classname = AuthFilter.class.getName(); final String pathSpec = WebHdfsFileSystem.PATH_PREFIX + "/*"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index ce4531c55e..148dc72d76 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -157,6 +157,8 @@ public synchronized void initialize(URI uri, Configuration conf ) throws IOException { super.initialize(uri, conf); setConf(conf); + /** set user pattern based on configuration file */ + UserParam.setUserPattern(conf.get(DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY, DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT)); connectionFactory = URLConnectionFactory .newDefaultURLConnectionFactory(conf); initializeTokenAspect(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/UserParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/UserParam.java index 36e128feed..481d8fea21 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/UserParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/UserParam.java @@ -17,8 +17,10 @@ */ package org.apache.hadoop.hdfs.web.resources; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT; import org.apache.hadoop.security.UserGroupInformation; - +import com.google.common.annotations.VisibleForTesting; + import java.text.MessageFormat; import java.util.regex.Pattern; @@ -29,8 +31,21 @@ public class UserParam extends StringParam { /** Default parameter value. */ public static final String DEFAULT = ""; - private static final Domain DOMAIN = new Domain(NAME, - Pattern.compile("^[A-Za-z_][A-Za-z0-9._-]*[$]?$")); + private static Domain domain = new Domain(NAME, Pattern.compile(DFS_WEBHDFS_USER_PATTERN_DEFAULT)); + + @VisibleForTesting + public static Domain getUserPatternDomain() { + return domain; + } + + @VisibleForTesting + public static void setUserPatternDomain(Domain dm) { + domain = dm; + } + + public static void setUserPattern(String pattern) { + domain = new Domain(NAME, Pattern.compile(pattern)); + } private static String validateLength(String str) { if (str == null) { @@ -50,7 +65,7 @@ private static String validateLength(String str) { * @param str a string representation of the parameter value. */ public UserParam(final String str) { - super(DOMAIN, str == null || str.equals(DEFAULT)? null : validateLength(str)); + super(domain, str == null || str.equals(DEFAULT)? null : validateLength(str)); } /** @@ -64,4 +79,4 @@ public UserParam(final UserGroupInformation ugi) { public String getName() { return NAME; } -} \ No newline at end of file +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 9eb9983ed1..78c14bb861 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -1593,4 +1593,12 @@ + + dfs.webhdfs.user.provider.user.pattern + ^[A-Za-z_][A-Za-z0-9._-]*[$]?$ + + Valid pattern for user and group names for webhdfs, it must be a valid java regex. + + + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java index e49f2268a6..a657cd8ba7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java @@ -261,6 +261,34 @@ public Void run() throws IOException, URISyntaxException { } } + @Test(timeout=300000) + public void testNumericalUserName() throws Exception { + final Configuration conf = WebHdfsTestUtil.createConf(); + conf.set(DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY, "^[A-Za-z0-9_][A-Za-z0-9._-]*[$]?$"); + final MiniDFSCluster cluster = + new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + try { + cluster.waitActive(); + WebHdfsTestUtil.getWebHdfsFileSystem(conf, WebHdfsFileSystem.SCHEME) + .setPermission(new Path("/"), + new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); + + UserGroupInformation.createUserForTesting("123", new String[]{"my-group"}) + .doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws IOException, URISyntaxException { + FileSystem fs = WebHdfsTestUtil.getWebHdfsFileSystem(conf, + WebHdfsFileSystem.SCHEME); + Path d = new Path("/my-dir"); + Assert.assertTrue(fs.mkdirs(d)); + return null; + } + }); + } finally { + cluster.shutdown(); + } + } + /** * WebHdfs should be enabled by default after HDFS-5532 * diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java index 97223ab2fb..8818f31965 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java @@ -285,4 +285,19 @@ public void testConcatSourcesParam() { Assert.assertEquals(expected, computed.getValue()); } } + + @Test + public void testUserNameOkAfterResettingPattern() { + UserParam.Domain oldDomain = UserParam.getUserPatternDomain(); + + String newPattern = "^[A-Za-z0-9_][A-Za-z0-9._-]*[$]?$"; + UserParam.setUserPattern(newPattern); + + UserParam userParam = new UserParam("1x"); + assertNotNull(userParam.getValue()); + userParam = new UserParam("123"); + assertNotNull(userParam.getValue()); + + UserParam.setUserPatternDomain(oldDomain); + } }