big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10148. Add Unit test for queue ACL for both FS and CS. Contributed by Kinga Marton

Browse Source
This commit is contained in:
Szilard Nemeth 2020-02-27 20:07:32 +01:00
parent 429da635ec
commit 10461e0193
4 changed files with 283 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java
View File

@ -113,5 +113,9 @@ public ApplicationClientProtocol run() throws Exception {
return userClient;
}
public Configuration getConf() {
return conf;
}
protected abstract Configuration createConfiguration() throws IOException;
}

146
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/QueueACLsTestBase.java
View File

@ -18,9 +18,12 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.junit.Assert;
import org.apache.hadoop.security.authorize.AccessControlList;
@ -43,6 +46,19 @@
public abstract class QueueACLsTestBase extends ACLsTestBase {
protected static final String QUEUED = "D";
protected static final String QUEUED1 = "D1";
private static final String ALL_ACL = "*";
private static final String NONE_ACL = " ";
abstract public String getQueueD();
abstract public String getQueueD1();
abstract public void updateConfigWithDAndD1Queues(String rootAcl,
String queueDAcl, String queueD1Acl) throws IOException;
@After
public void tearDown() {
if (resourceManager != null) {
@ -75,6 +91,136 @@ public void testApplicationACLs() throws Exception {
}
/**
* Test for the case when the following submit application
* and administer queue ACLs are defined:
* root: (none)
* D: * (all)
* D1: * (all)
* Expected result: the user will have access only to D and D1 queues.
* @throws IOException
*/
@Test
public void testQueueAclRestrictedRootACL() throws IOException {
updateConfigWithDAndD1Queues(NONE_ACL, ALL_ACL, ALL_ACL);
checkAccess(false, true, true);
}
/**
* Test for the case when the following submit application
* and administer queue ACLs are defined:
* root: (none)
* D: (none)
* D1: (none)
* Expected result: the user will have to none of the queues.
* @throws IOException
*/
@Test
public void testQueueAclNoAccess() throws IOException {
updateConfigWithDAndD1Queues(NONE_ACL, NONE_ACL, NONE_ACL);
checkAccess(false, false, false);
}
/**
* Test for the case when the following submit application
* and administer queue ACLs are defined:
* root: (none)
* D: * (all)
* D1: (none)
* Expected result: access to D1 will be permitted by root.D,
* so the user will be able to access queues D and D1.
* @throws IOException
*/
@Test
public void testQueueAclRestrictedRootAndD1() throws IOException {
updateConfigWithDAndD1Queues(NONE_ACL, ALL_ACL, NONE_ACL);
checkAccess(false, true, true);
}
/**
* Test for the case when the following submit application
* and administer queue ACLs are defined:
* root: (none)
* D: (none)
* D1: (all)
* Expected result: only queue D1 can be accessed.
* @throws IOException
*/
@Test
public void testQueueAclRestrictedRootAndD() throws IOException {
updateConfigWithDAndD1Queues(NONE_ACL, NONE_ACL, ALL_ACL);
checkAccess(false, false, true);
}
/**
* Test for the case when the following submit application
* and administer queue ACLs are defined:
* root: * (all)
* D: (none)
* D1: * (all)
* Expected result: access to D will be permitted from the root queue,
* so the user will be able to access queues root, D and D1.
* @throws IOException
*/
@Test
public void testQueueAclRestrictedD() throws IOException {
updateConfigWithDAndD1Queues(ALL_ACL, NONE_ACL, ALL_ACL);
checkAccess(true, true, true);
}
/**
* Test for the case when the following submit application
* and administer queue ACLs are defined:
* root: * (all)
* D: * (all)
* D1: (none)
* Expected result: access to D1 will be permitted from queue D,
* so the user will be able to access queues root, D and D1.
* @throws IOException
*/
@Test
public void testQueueAclRestrictedD1() throws IOException {
updateConfigWithDAndD1Queues(ALL_ACL, ALL_ACL, NONE_ACL);
checkAccess(true, true, true);
}
/**
* Test for the case when no ACLs are defined, so the default values are used
* Expected result: The default ACLs for the root queue is "*"(all) and for
* the other queues are " " (none), so the user will have access to all the
* queues because they will have permissions from the root.
*
* @throws IOException
*/
@Test
public void testQueueAclDefaultValues() throws IOException {
updateConfigWithDAndD1Queues(null, null, null);
checkAccess(true, true, true);
}
private void checkAccess(boolean rootAccess, boolean dAccess,
boolean d1Access)throws IOException {
checkAccess(rootAccess, "root");
checkAccess(dAccess, getQueueD());
checkAccess(d1Access, getQueueD1());
}
private void checkAccess(boolean access, String queueName)
throws IOException {
UserGroupInformation user = UserGroupInformation.getCurrentUser();
String failureMsg = "Wrong %s access to %s queue";
Assert.assertEquals(
String.format(failureMsg, QueueACL.ADMINISTER_QUEUE, queueName),
access, resourceManager.getResourceScheduler()
.checkAccess(user, QueueACL.ADMINISTER_QUEUE, queueName));
Assert.assertEquals(
String.format(failureMsg, QueueACL.SUBMIT_APPLICATIONS, queueName),
access, resourceManager.getResourceScheduler()
.checkAccess(user, QueueACL.SUBMIT_APPLICATIONS, queueName));
}
private void verifyGetClientAMToken(String submitter, String queueAdmin,
String queueName, boolean setupACLs) throws Exception {
ApplicationId applicationId =

75
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueACLs.java
View File

@ -17,6 +17,7 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
@ -34,8 +35,10 @@ protected Configuration createConfiguration() {
csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {
QUEUEA, QUEUEB });
csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEA, 50f);
csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEB, 50f);
setQueueCapacity(csConf, 50,
CapacitySchedulerConfiguration.ROOT + "." + QUEUEA);
setQueueCapacity(csConf, 50,
CapacitySchedulerConfiguration.ROOT + "." + QUEUEB);
Map<QueueACL, AccessControlList> aclsOnQueueA =
new HashMap<QueueACL, AccessControlList>();
@ -71,4 +74,72 @@ protected Configuration createConfiguration() {
return csConf;
}
@Override
public String getQueueD() {
return QUEUED;
}
@Override
public String getQueueD1() {
return QUEUED1;
}
/**
* Updates the configuration with the following queue hierarchy:
* root
* |
* D
* |
* D1.
* @param rootAcl administer queue and submit application ACL for root queue
* @param queueDAcl administer queue and submit application ACL for D queue
* @param queueD1Acl administer queue and submit application ACL for D1 queue
* @throws IOException
*/
@Override
public void updateConfigWithDAndD1Queues(String rootAcl, String queueDAcl,
String queueD1Acl) throws IOException {
CapacitySchedulerConfiguration csConf =
(CapacitySchedulerConfiguration) getConf();
csConf.clear();
csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
new String[] {QUEUED, QUEUEA, QUEUEB});
String dPath = CapacitySchedulerConfiguration.ROOT + "." + QUEUED;
String d1Path = dPath + "." + QUEUED1;
csConf.setQueues(dPath, new String[] {QUEUED1});
setQueueCapacity(csConf, 100, d1Path);
setQueueCapacity(csConf, 30, CapacitySchedulerConfiguration.ROOT
+ "." + QUEUEA);
setQueueCapacity(csConf, 50, CapacitySchedulerConfiguration.ROOT
+ "." + QUEUEB);
setQueueCapacity(csConf, 20, dPath);
if (rootAcl != null) {
setAdminAndSubmitACL(csConf, rootAcl,
CapacitySchedulerConfiguration.ROOT);
}
if (queueDAcl != null) {
setAdminAndSubmitACL(csConf, queueDAcl, dPath);
}
if (queueD1Acl != null) {
setAdminAndSubmitACL(csConf, d1Path, queueD1Acl);
}
resourceManager.getResourceScheduler()
.reinitialize(csConf, resourceManager.getRMContext());
}
private void setQueueCapacity(CapacitySchedulerConfiguration csConf,
float capacity, String queuePath) {
csConf.setCapacity(queuePath, capacity);
}
private void setAdminAndSubmitACL(CapacitySchedulerConfiguration csConf,
String queueAcl, String queuePath) {
csConf.setAcl(queuePath, QueueACL.ADMINISTER_QUEUE, queueAcl);
csConf.setAcl(queuePath, QueueACL.SUBMIT_APPLICATIONS, queueAcl);
}
}

62
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java
View File

@ -18,8 +18,10 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import java.io.File;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.QueueACLsTestBase;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
@ -32,8 +34,8 @@ public class TestFairSchedulerQueueACLs extends QueueACLsTestBase {
protected Configuration createConfiguration() {
FairSchedulerConfiguration fsConf = new FairSchedulerConfiguration();
final String testDir = new File(System.getProperty("test.build.data",
"/tmp")).getAbsolutePath();
final String testDir = new File(System.getProperty(
GenericTestUtils.SYSPROP_TEST_DATA_DIR, "/tmp")).getAbsolutePath();
final String allocFile = new File(testDir, "test-queues.xml")
.getAbsolutePath();
@ -57,4 +59,60 @@ protected Configuration createConfiguration() {
return fsConf;
}
@Override
public String getQueueD() {
return "root." + QUEUED;
}
@Override
public String getQueueD1() {
return "root."+ QUEUED + "." + QUEUED1;
}
/**
* Creates the following queue hierarchy:
* root
* |
* D
* |
* D1.
* @param rootAcl administer queue and submit application ACL for root queue
* @param queueDAcl administer queue and submit application ACL for D queue
* @param queueD1Acl administer queue and submit application ACL for D1 queue
* @throws IOException
*/
@Override
public void updateConfigWithDAndD1Queues(String rootAcl, String queueDAcl,
String queueD1Acl) throws IOException {
FairSchedulerConfiguration fsConf = (FairSchedulerConfiguration) getConf();
fsConf.clear();
final String testDir = new File(System.getProperty(
GenericTestUtils.SYSPROP_TEST_DATA_DIR, "/tmp")).getAbsolutePath();
final String allocFile = new File(testDir, "test-queues.xml")
.getAbsolutePath();
AllocationFileWriter.create()
.addQueue(new AllocationFileQueue.Builder("root")
.aclSubmitApps(rootAcl)
.aclAdministerApps(rootAcl)
.subQueue(new AllocationFileQueue.Builder(QUEUED)
.aclAdministerApps(queueDAcl)
.aclSubmitApps(queueDAcl)
.subQueue(new AllocationFileQueue.Builder(QUEUED1)
.aclSubmitApps(queueD1Acl)
.aclAdministerApps(queueD1Acl)
.build())
.build())
.build())
.writeToFile(allocFile);
fsConf.set(FairSchedulerConfiguration.ALLOCATION_FILE, allocFile);
fsConf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
fsConf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName());
resourceManager.getResourceScheduler()
.reinitialize(fsConf, resourceManager.getRMContext());
}
}