From 8a615259281511de639c1f9aa719a842e34a523f Mon Sep 17 00:00:00 2001 From: Robert Kanter Date: Mon, 16 Oct 2017 15:34:32 -0700 Subject: [PATCH] YARN-7308. TestApplicationACLs fails with FairScheduler (rkanter) --- .../ParameterizedSchedulerTestBase.java | 3 +- .../resourcemanager/TestApplicationACLs.java | 90 ++++++++++++++----- 2 files changed, 69 insertions(+), 24 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java index 289ff1cebe..9a29a89657 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java @@ -89,7 +89,8 @@ public ParameterizedSchedulerTestBase(SchedulerType type) } } - private void configureFairScheduler(YarnConfiguration conf) throws IOException { + protected void configureFairScheduler(YarnConfiguration conf) + throws IOException { // Disable queueMaxAMShare limitation for fair scheduler PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE)); out.println(""); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java index cab1679faf..c7ed02c9c2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java @@ -22,7 +22,10 @@ import static org.mockito.Mockito.when; import static org.mockito.Matchers.any; +import java.io.File; +import java.io.FileWriter; import java.io.IOException; +import java.io.PrintWriter; import java.net.InetSocketAddress; import java.security.PrivilegedExceptionAction; import java.util.HashMap; @@ -30,6 +33,9 @@ import java.util.Map; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; +import org.junit.After; import org.junit.Assert; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -57,19 +63,17 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.YarnRPC; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStoreFactory; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; -import org.junit.AfterClass; -import org.junit.BeforeClass; +import org.junit.Before; import org.junit.Test; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -public class TestApplicationACLs { +public class TestApplicationACLs extends ParameterizedSchedulerTestBase { private static final String APP_OWNER = "owner"; private static final String FRIEND = "friend"; @@ -82,27 +86,34 @@ public class TestApplicationACLs { private static final Log LOG = LogFactory.getLog(TestApplicationACLs.class); - static MockRM resourceManager; - static Configuration conf = new YarnConfiguration(); - final static YarnRPC rpc = YarnRPC.create(conf); - final static InetSocketAddress rmAddress = conf.getSocketAddr( - YarnConfiguration.RM_ADDRESS, - YarnConfiguration.DEFAULT_RM_ADDRESS, - YarnConfiguration.DEFAULT_RM_PORT); - private static ApplicationClientProtocol rmClient; + private MockRM resourceManager; + private Configuration conf; + private YarnRPC rpc; + private InetSocketAddress rmAddress; + private ApplicationClientProtocol rmClient; + private RecordFactory recordFactory; + private boolean isQueueUser; - private static RecordFactory recordFactory = RecordFactoryProvider - .getRecordFactory(conf); + public TestApplicationACLs(SchedulerType type) throws IOException { + super(type); + } - private static boolean isQueueUser = false; - - @BeforeClass - public static void setup() throws InterruptedException, IOException { - RMStateStore store = RMStateStoreFactory.getStore(conf); + @Before + public void setup() throws InterruptedException, IOException { + conf = getConf(); + rpc = YarnRPC.create(conf); + rmAddress = conf.getSocketAddr( + YarnConfiguration.RM_ADDRESS, + YarnConfiguration.DEFAULT_RM_ADDRESS, + YarnConfiguration.DEFAULT_RM_PORT); + RMStateStoreFactory.getStore(conf); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); AccessControlList adminACL = new AccessControlList(""); adminACL.addGroup(SUPER_GROUP); conf.set(YarnConfiguration.YARN_ADMIN_ACL, adminACL.getAclString()); + recordFactory = RecordFactoryProvider + .getRecordFactory(conf); + isQueueUser = false; resourceManager = new MockRM(conf) { @@ -161,13 +172,36 @@ public ApplicationClientProtocol run() throws Exception { }); } - @AfterClass - public static void tearDown() { + @After + public void tearDown() { if(resourceManager != null) { resourceManager.stop(); } } + @Override + protected void configureFairScheduler(YarnConfiguration conf) + throws IOException { + final String testDir = new File(System.getProperty("test.build.data", + "/tmp")).getAbsolutePath(); + final String allocFile = new File(testDir, "test-queues.xml") + .getAbsolutePath(); + PrintWriter out = new PrintWriter(new FileWriter(allocFile)); + out.println(""); + out.println(""); + out.println(""); + out.println(" "); + out.println(" "); + out.println(""); + out.println(""); + out.println(" "); + out.println(" "); + out.println(""); + out.println(""); + out.close(); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, allocFile); + } + @Test public void testApplicationACLs() throws Exception { @@ -207,6 +241,10 @@ private ApplicationId submitAppAndGetAppId(AccessControlList viewACL, Resource resource = BuilderUtils.newResource(1024, 1); context.setResource(resource); amContainer.setApplicationACLs(acls); + if (conf.get(YarnConfiguration.RM_SCHEDULER) + .equals(FairScheduler.class.getName())) { + context.setQueue("root.default"); + } context.setAMContainerSpec(amContainer); submitRequest.setApplicationSubmissionContext(context); rmClient.submitApplication(submitRequest); @@ -424,8 +462,14 @@ private void verifyInvalidQueueWithAcl() throws Exception { GetApplicationReportResponse applicationReport = rmClient.getApplicationReport(appReportRequest); ApplicationReport appReport = applicationReport.getApplicationReport(); - Assert.assertTrue(appReport.getDiagnostics() - .contains("submitted by user owner to unknown queue: InvalidQueue")); + if (conf.get(YarnConfiguration.RM_SCHEDULER) + .equals(FairScheduler.class.getName())) { + Assert.assertTrue(appReport.getDiagnostics() + .contains("Application rejected by queue placement policy")); + } else { + Assert.assertTrue(appReport.getDiagnostics() + .contains("submitted by user owner to unknown queue: InvalidQueue")); + } } private void verifyAdministerQueueUserAccess() throws Exception {