HDFS-11421. Make WebHDFS' ACLs RegEx configurable. Contributed by Harsh J.
This commit is contained in:
parent
d2b3ba9b8f
commit
e24ed47d9a
@ -35,6 +35,8 @@ public interface HdfsClientConfigKeys {
|
|||||||
String DFS_WEBHDFS_USER_PATTERN_KEY =
|
String DFS_WEBHDFS_USER_PATTERN_KEY =
|
||||||
"dfs.webhdfs.user.provider.user.pattern";
|
"dfs.webhdfs.user.provider.user.pattern";
|
||||||
String DFS_WEBHDFS_USER_PATTERN_DEFAULT = "^[A-Za-z_][A-Za-z0-9._-]*[$]?$";
|
String DFS_WEBHDFS_USER_PATTERN_DEFAULT = "^[A-Za-z_][A-Za-z0-9._-]*[$]?$";
|
||||||
|
String DFS_WEBHDFS_ACL_PERMISSION_PATTERN_KEY =
|
||||||
|
"dfs.webhdfs.acl.provider.permission.pattern";
|
||||||
String DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT =
|
String DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT =
|
||||||
"^(default:)?(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3})?(,(default:)?(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3})?)*$";
|
"^(default:)?(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3})?(,(default:)?(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3})?)*$";
|
||||||
|
|
||||||
|
@ -183,10 +183,14 @@ public synchronized void initialize(URI uri, Configuration conf
|
|||||||
) throws IOException {
|
) throws IOException {
|
||||||
super.initialize(uri, conf);
|
super.initialize(uri, conf);
|
||||||
setConf(conf);
|
setConf(conf);
|
||||||
/** set user pattern based on configuration file */
|
|
||||||
|
// set user and acl patterns based on configuration file
|
||||||
UserParam.setUserPattern(conf.get(
|
UserParam.setUserPattern(conf.get(
|
||||||
HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY,
|
HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY,
|
||||||
HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT));
|
HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT));
|
||||||
|
AclPermissionParam.setAclPermissionPattern(conf.get(
|
||||||
|
HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_KEY,
|
||||||
|
HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT));
|
||||||
|
|
||||||
boolean isOAuth = conf.getBoolean(
|
boolean isOAuth = conf.getBoolean(
|
||||||
HdfsClientConfigKeys.DFS_WEBHDFS_OAUTH_ENABLED_KEY,
|
HdfsClientConfigKeys.DFS_WEBHDFS_OAUTH_ENABLED_KEY,
|
||||||
|
@ -24,6 +24,7 @@
|
|||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.regex.Pattern;
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
import org.apache.hadoop.fs.permission.AclEntry;
|
import org.apache.hadoop.fs.permission.AclEntry;
|
||||||
|
|
||||||
/** AclPermission parameter. */
|
/** AclPermission parameter. */
|
||||||
@ -33,7 +34,7 @@ public class AclPermissionParam extends StringParam {
|
|||||||
/** Default parameter value. */
|
/** Default parameter value. */
|
||||||
public static final String DEFAULT = "";
|
public static final String DEFAULT = "";
|
||||||
|
|
||||||
private static final Domain DOMAIN = new Domain(NAME,
|
private static Domain DOMAIN = new Domain(NAME,
|
||||||
Pattern.compile(DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT));
|
Pattern.compile(DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT));
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -49,6 +50,20 @@ public AclPermissionParam(List<AclEntry> acl) {
|
|||||||
super(DOMAIN,parseAclSpec(acl).equals(DEFAULT) ? null : parseAclSpec(acl));
|
super(DOMAIN,parseAclSpec(acl).equals(DEFAULT) ? null : parseAclSpec(acl));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@VisibleForTesting
|
||||||
|
public static Domain getAclPermissionPattern() {
|
||||||
|
return DOMAIN;
|
||||||
|
}
|
||||||
|
|
||||||
|
@VisibleForTesting
|
||||||
|
public static void setAclPermissionPattern(Domain dm) {
|
||||||
|
DOMAIN = dm;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void setAclPermissionPattern(String pattern) {
|
||||||
|
DOMAIN = new Domain(NAME, Pattern.compile(pattern));
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String getName() {
|
public String getName() {
|
||||||
return NAME;
|
return NAME;
|
||||||
|
@ -48,11 +48,12 @@
|
|||||||
import org.apache.hadoop.fs.permission.FsCreateModes;
|
import org.apache.hadoop.fs.permission.FsCreateModes;
|
||||||
import org.apache.hadoop.fs.permission.FsPermission;
|
import org.apache.hadoop.fs.permission.FsPermission;
|
||||||
import org.apache.hadoop.hdfs.DFSClient;
|
import org.apache.hadoop.hdfs.DFSClient;
|
||||||
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
|
||||||
import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
|
import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
|
||||||
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
|
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
|
||||||
import org.apache.hadoop.hdfs.web.JsonUtil;
|
import org.apache.hadoop.hdfs.web.JsonUtil;
|
||||||
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
|
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
|
||||||
|
import org.apache.hadoop.hdfs.web.resources.AclPermissionParam;
|
||||||
import org.apache.hadoop.hdfs.web.resources.GetOpParam;
|
import org.apache.hadoop.hdfs.web.resources.GetOpParam;
|
||||||
import org.apache.hadoop.hdfs.web.resources.PostOpParam;
|
import org.apache.hadoop.hdfs.web.resources.PostOpParam;
|
||||||
import org.apache.hadoop.hdfs.web.resources.PutOpParam;
|
import org.apache.hadoop.hdfs.web.resources.PutOpParam;
|
||||||
@ -112,8 +113,11 @@ public WebHdfsHandler(Configuration conf, Configuration confForCreate)
|
|||||||
this.confForCreate = confForCreate;
|
this.confForCreate = confForCreate;
|
||||||
/** set user pattern based on configuration file */
|
/** set user pattern based on configuration file */
|
||||||
UserParam.setUserPattern(
|
UserParam.setUserPattern(
|
||||||
conf.get(DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY,
|
conf.get(HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY,
|
||||||
DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT));
|
HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT));
|
||||||
|
AclPermissionParam.setAclPermissionPattern(
|
||||||
|
conf.get(HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_KEY,
|
||||||
|
HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -41,6 +41,7 @@
|
|||||||
import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods;
|
import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods;
|
||||||
import org.apache.hadoop.hdfs.web.AuthFilter;
|
import org.apache.hadoop.hdfs.web.AuthFilter;
|
||||||
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
|
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
|
||||||
|
import org.apache.hadoop.hdfs.web.resources.AclPermissionParam;
|
||||||
import org.apache.hadoop.hdfs.web.resources.Param;
|
import org.apache.hadoop.hdfs.web.resources.Param;
|
||||||
import org.apache.hadoop.hdfs.web.resources.UserParam;
|
import org.apache.hadoop.hdfs.web.resources.UserParam;
|
||||||
import org.apache.hadoop.http.HttpConfig;
|
import org.apache.hadoop.http.HttpConfig;
|
||||||
@ -80,6 +81,9 @@ private void initWebHdfs(Configuration conf) throws IOException {
|
|||||||
UserParam.setUserPattern(conf.get(
|
UserParam.setUserPattern(conf.get(
|
||||||
HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY,
|
HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY,
|
||||||
HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT));
|
HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_DEFAULT));
|
||||||
|
AclPermissionParam.setAclPermissionPattern(conf.get(
|
||||||
|
HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_KEY,
|
||||||
|
HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT));
|
||||||
|
|
||||||
// add authentication filter for webhdfs
|
// add authentication filter for webhdfs
|
||||||
final String className = conf.get(
|
final String className = conf.get(
|
||||||
|
@ -2477,6 +2477,14 @@
|
|||||||
</description>
|
</description>
|
||||||
</property>
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>dfs.webhdfs.acl.provider.permission.pattern</name>
|
||||||
|
<value>^(default:)?(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3})?(,(default:)?(user|group|mask|other):[[A-Za-z_][A-Za-z0-9._-]]*:([rwx-]{3})?)*$</value>
|
||||||
|
<description>
|
||||||
|
Valid pattern for user and group names in webhdfs acl operations, it must be a valid java regex.
|
||||||
|
</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
<property>
|
<property>
|
||||||
<name>dfs.webhdfs.socket.connect-timeout</name>
|
<name>dfs.webhdfs.socket.connect-timeout</name>
|
||||||
<value>60s</value>
|
<value>60s</value>
|
||||||
|
@ -49,6 +49,7 @@
|
|||||||
import javax.servlet.http.HttpServletResponse;
|
import javax.servlet.http.HttpServletResponse;
|
||||||
import javax.ws.rs.core.MediaType;
|
import javax.ws.rs.core.MediaType;
|
||||||
|
|
||||||
|
import com.google.common.collect.ImmutableList;
|
||||||
import org.apache.commons.io.IOUtils;
|
import org.apache.commons.io.IOUtils;
|
||||||
import org.apache.commons.logging.Log;
|
import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
@ -64,6 +65,9 @@
|
|||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.fs.RemoteIterator;
|
import org.apache.hadoop.fs.RemoteIterator;
|
||||||
import org.apache.hadoop.fs.StorageType;
|
import org.apache.hadoop.fs.StorageType;
|
||||||
|
import org.apache.hadoop.fs.permission.AclEntry;
|
||||||
|
import org.apache.hadoop.fs.permission.AclEntryScope;
|
||||||
|
import org.apache.hadoop.fs.permission.AclEntryType;
|
||||||
import org.apache.hadoop.fs.permission.FsAction;
|
import org.apache.hadoop.fs.permission.FsAction;
|
||||||
import org.apache.hadoop.fs.permission.FsPermission;
|
import org.apache.hadoop.fs.permission.FsPermission;
|
||||||
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
||||||
@ -379,10 +383,17 @@ public Void run() throws IOException, URISyntaxException {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Test(timeout=300000)
|
@Test(timeout=300000)
|
||||||
public void testNumericalUserName() throws Exception {
|
public void testCustomizedUserAndGroupNames() throws Exception {
|
||||||
final Configuration conf = WebHdfsTestUtil.createConf();
|
final Configuration conf = WebHdfsTestUtil.createConf();
|
||||||
|
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
|
||||||
|
// Modify username pattern to allow numeric usernames
|
||||||
conf.set(HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY, "^[A-Za-z0-9_][A-Za-z0-9" +
|
conf.set(HdfsClientConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY, "^[A-Za-z0-9_][A-Za-z0-9" +
|
||||||
"._-]*[$]?$");
|
"._-]*[$]?$");
|
||||||
|
// Modify acl pattern to allow numeric and "@" characters user/groups in ACL spec
|
||||||
|
conf.set(HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_KEY,
|
||||||
|
"^(default:)?(user|group|mask|other):" +
|
||||||
|
"[[0-9A-Za-z_][@A-Za-z0-9._-]]*:([rwx-]{3})?(,(default:)?" +
|
||||||
|
"(user|group|mask|other):[[0-9A-Za-z_][@A-Za-z0-9._-]]*:([rwx-]{3})?)*$");
|
||||||
final MiniDFSCluster cluster =
|
final MiniDFSCluster cluster =
|
||||||
new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
|
new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
|
||||||
try {
|
try {
|
||||||
@ -391,6 +402,7 @@ public void testNumericalUserName() throws Exception {
|
|||||||
.setPermission(new Path("/"),
|
.setPermission(new Path("/"),
|
||||||
new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL));
|
new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL));
|
||||||
|
|
||||||
|
// Test a numeric username
|
||||||
UserGroupInformation.createUserForTesting("123", new String[]{"my-group"})
|
UserGroupInformation.createUserForTesting("123", new String[]{"my-group"})
|
||||||
.doAs(new PrivilegedExceptionAction<Void>() {
|
.doAs(new PrivilegedExceptionAction<Void>() {
|
||||||
@Override
|
@Override
|
||||||
@ -399,6 +411,21 @@ public Void run() throws IOException, URISyntaxException {
|
|||||||
WebHdfsConstants.WEBHDFS_SCHEME);
|
WebHdfsConstants.WEBHDFS_SCHEME);
|
||||||
Path d = new Path("/my-dir");
|
Path d = new Path("/my-dir");
|
||||||
Assert.assertTrue(fs.mkdirs(d));
|
Assert.assertTrue(fs.mkdirs(d));
|
||||||
|
// Test also specifying a default ACL with a numeric username
|
||||||
|
// and another of a groupname with '@'
|
||||||
|
fs.modifyAclEntries(d, ImmutableList.of(
|
||||||
|
new AclEntry.Builder()
|
||||||
|
.setPermission(FsAction.READ)
|
||||||
|
.setScope(AclEntryScope.DEFAULT)
|
||||||
|
.setType(AclEntryType.USER)
|
||||||
|
.setName("11010")
|
||||||
|
.build(),
|
||||||
|
new AclEntry.Builder()
|
||||||
|
.setPermission(FsAction.READ_WRITE)
|
||||||
|
.setType(AclEntryType.GROUP)
|
||||||
|
.setName("foo@bar")
|
||||||
|
.build()
|
||||||
|
));
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
@ -351,6 +351,40 @@ public void testAclPermissionParam() {
|
|||||||
LOG.info("EXPECTED: " + e);
|
LOG.info("EXPECTED: " + e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testUserGroupOkAfterAlteringAclPattern() {
|
||||||
|
// Preserve default pattern value
|
||||||
|
AclPermissionParam.Domain oldDomain =
|
||||||
|
AclPermissionParam.getAclPermissionPattern();
|
||||||
|
|
||||||
|
// Override the pattern with one that accepts '@' and numbers
|
||||||
|
// in the first character of usernames/groupnames
|
||||||
|
String newPattern =
|
||||||
|
"^(default:)?(user|group|mask|other):" +
|
||||||
|
"[[0-9A-Za-z_][@A-Za-z0-9._-]]*:([rwx-]{3})?" +
|
||||||
|
"(,(default:)?(user|group|mask|other):" +
|
||||||
|
"[[0-9A-Za-z_][@A-Za-z0-9._-]]*:([rwx-]{3})?)*$";
|
||||||
|
|
||||||
|
try {
|
||||||
|
AclPermissionParam.setAclPermissionPattern(newPattern);
|
||||||
|
|
||||||
|
String numericUserSpec = "user:110201:rwx";
|
||||||
|
AclPermissionParam aclNumericUserParam =
|
||||||
|
new AclPermissionParam(numericUserSpec);
|
||||||
|
Assert.assertEquals(numericUserSpec, aclNumericUserParam.getValue());
|
||||||
|
|
||||||
|
String oddGroupSpec = "group:foo@bar:rwx";
|
||||||
|
AclPermissionParam aclGroupWithDomainParam =
|
||||||
|
new AclPermissionParam(oddGroupSpec);
|
||||||
|
Assert.assertEquals(oddGroupSpec, aclGroupWithDomainParam.getValue());
|
||||||
|
|
||||||
|
} finally {
|
||||||
|
// Revert back to the default rules for remainder of tests
|
||||||
|
AclPermissionParam.setAclPermissionPattern(oldDomain);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testXAttrNameParam() {
|
public void testXAttrNameParam() {
|
||||||
|
Loading…
Reference in New Issue
Block a user