big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-7308. TestApplicationACLs fails with FairScheduler (rkanter)

Browse Source
This commit is contained in:
Robert Kanter 2017-10-16 15:34:32 -07:00
parent 4540ffd15f
commit 8a61525928
2 changed files with 69 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java
View File

@ -89,7 +89,8 @@ public ParameterizedSchedulerTestBase(SchedulerType type)
}
}
private void configureFairScheduler(YarnConfiguration conf) throws IOException {
protected void configureFairScheduler(YarnConfiguration conf)
throws IOException {
// Disable queueMaxAMShare limitation for fair scheduler
PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE));
out.println("<?xml version=\"1.0\"?>");

90
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
View File

@ -22,7 +22,10 @@
import static org.mockito.Mockito.when;
import static org.mockito.Matchers.any;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.InetSocketAddress;
import java.security.PrivilegedExceptionAction;
import java.util.HashMap;
@ -30,6 +33,9 @@
import java.util.Map;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
import org.junit.After;
import org.junit.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -57,19 +63,17 @@
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStoreFactory;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Before;
import org.junit.Test;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
public class TestApplicationACLs {
public class TestApplicationACLs extends ParameterizedSchedulerTestBase {
private static final String APP_OWNER = "owner";
private static final String FRIEND = "friend";
@ -82,27 +86,34 @@ public class TestApplicationACLs {
private static final Log LOG = LogFactory.getLog(TestApplicationACLs.class);
static MockRM resourceManager;
static Configuration conf = new YarnConfiguration();
final static YarnRPC rpc = YarnRPC.create(conf);
final static InetSocketAddress rmAddress = conf.getSocketAddr(
YarnConfiguration.RM_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADDRESS,
YarnConfiguration.DEFAULT_RM_PORT);
private static ApplicationClientProtocol rmClient;
private MockRM resourceManager;
private Configuration conf;
private YarnRPC rpc;
private InetSocketAddress rmAddress;
private ApplicationClientProtocol rmClient;
private RecordFactory recordFactory;
private boolean isQueueUser;
private static RecordFactory recordFactory = RecordFactoryProvider
.getRecordFactory(conf);
public TestApplicationACLs(SchedulerType type) throws IOException {
super(type);
}
private static boolean isQueueUser = false;
@BeforeClass
public static void setup() throws InterruptedException, IOException {
RMStateStore store = RMStateStoreFactory.getStore(conf);
@Before
public void setup() throws InterruptedException, IOException {
conf = getConf();
rpc = YarnRPC.create(conf);
rmAddress = conf.getSocketAddr(
YarnConfiguration.RM_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADDRESS,
YarnConfiguration.DEFAULT_RM_PORT);
RMStateStoreFactory.getStore(conf);
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
AccessControlList adminACL = new AccessControlList("");
adminACL.addGroup(SUPER_GROUP);
conf.set(YarnConfiguration.YARN_ADMIN_ACL, adminACL.getAclString());
recordFactory = RecordFactoryProvider
.getRecordFactory(conf);
isQueueUser = false;
resourceManager = new MockRM(conf) {
@ -161,13 +172,36 @@ public ApplicationClientProtocol run() throws Exception {
});
}
@AfterClass
public static void tearDown() {
@After
public void tearDown() {
if(resourceManager != null) {
resourceManager.stop();
}
}
@Override
protected void configureFairScheduler(YarnConfiguration conf)
throws IOException {
final String testDir = new File(System.getProperty("test.build.data",
"/tmp")).getAbsolutePath();
final String allocFile = new File(testDir, "test-queues.xml")
.getAbsolutePath();
PrintWriter out = new PrintWriter(new FileWriter(allocFile));
out.println("<?xml version=\"1.0\"?>");
out.println("<allocations>");
out.println("<queue name=\"root\" >");
out.println(" <queue name=\"default\">");
out.println(" </queue>");
out.println("</queue>");
out.println("<queuePlacementPolicy>");
out.println(" <rule name=\"specified\" create=\"false\" />");
out.println(" <rule name=\"reject\" />");
out.println("</queuePlacementPolicy>");
out.println("</allocations>");
out.close();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, allocFile);
}
@Test
public void testApplicationACLs() throws Exception {
@ -207,6 +241,10 @@ private ApplicationId submitAppAndGetAppId(AccessControlList viewACL,
Resource resource = BuilderUtils.newResource(1024, 1);
context.setResource(resource);
amContainer.setApplicationACLs(acls);
if (conf.get(YarnConfiguration.RM_SCHEDULER)
.equals(FairScheduler.class.getName())) {
context.setQueue("root.default");
}
context.setAMContainerSpec(amContainer);
submitRequest.setApplicationSubmissionContext(context);
rmClient.submitApplication(submitRequest);
@ -424,8 +462,14 @@ private void verifyInvalidQueueWithAcl() throws Exception {
GetApplicationReportResponse applicationReport =
rmClient.getApplicationReport(appReportRequest);
ApplicationReport appReport = applicationReport.getApplicationReport();
Assert.assertTrue(appReport.getDiagnostics()
.contains("submitted by user owner to unknown queue: InvalidQueue"));
if (conf.get(YarnConfiguration.RM_SCHEDULER)
.equals(FairScheduler.class.getName())) {
Assert.assertTrue(appReport.getDiagnostics()
.contains("Application rejected by queue placement policy"));
} else {
Assert.assertTrue(appReport.getDiagnostics()
.contains("submitted by user owner to unknown queue: InvalidQueue"));
}
}
private void verifyAdministerQueueUserAccess() throws Exception {