From 2db1d4d4ee7838f2a8c0b3999b6055456798321d Mon Sep 17 00:00:00 2001 From: Sanford Ryza Date: Thu, 3 Oct 2013 01:17:48 +0000 Subject: [PATCH] YARN-1213. Restore config to ban submitting to undeclared pools in the Fair Scheduler. (Sandy Ryza) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1528696 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../scheduler/fair/FairScheduler.java | 7 +- .../fair/FairSchedulerConfiguration.java | 9 ++ .../scheduler/fair/QueueManager.java | 22 ++--- .../scheduler/fair/TestFairScheduler.java | 88 ++++++++++++++----- 5 files changed, 89 insertions(+), 40 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 2de54a1781..5efa8d158d 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -92,6 +92,9 @@ Release 2.1.2 - UNRELEASED YARN-1228. Clean up Fair Scheduler configuration loading. (Sandy Ryza) + YARN-1213. Restore config to ban submitting to undeclared pools in the + Fair Scheduler. (Sandy Ryza) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index fa4c21805b..f7897e0f2f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -677,10 +677,11 @@ FSLeafQueue assignToQueue(RMApp rmApp, String queueName, String user) { queueName = user; } - FSLeafQueue queue = queueMgr.getLeafQueue(queueName); + FSLeafQueue queue = queueMgr.getLeafQueue(queueName, + conf.getAllowUndeclaredPools()); if (queue == null) { // queue is not an existing or createable leaf queue - queue = queueMgr.getLeafQueue(YarnConfiguration.DEFAULT_QUEUE_NAME); + queue = queueMgr.getLeafQueue(YarnConfiguration.DEFAULT_QUEUE_NAME, false); } if (rmApp != null) { @@ -726,7 +727,7 @@ private synchronized void removeApplication( // Inform the queue FSLeafQueue queue = queueMgr.getLeafQueue(application.getQueue() - .getQueueName()); + .getQueueName(), false); queue.removeApp(application); // Remove from our data-structure diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java index edfc8fa83e..b76d8eb8d7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java @@ -54,6 +54,11 @@ public class FairSchedulerConfiguration extends Configuration { protected static final String DEFAULT_ALLOCATION_FILE = "fair-scheduler.xml"; protected static final String EVENT_LOG_DIR = "eventlog.dir"; + /** Whether pools can be created that were not specified in the FS configuration file + */ + protected static final String ALLOW_UNDECLARED_POOLS = CONF_PREFIX + "allow-undeclared-pools"; + protected static final boolean DEFAULT_ALLOW_UNDECLARED_POOLS = true; + /** Whether to use the user name as the queue name (instead of "default") if * the request does not specify a queue. */ protected static final String USER_AS_DEFAULT_QUEUE = CONF_PREFIX + "user-as-default-queue"; @@ -141,6 +146,10 @@ public Resource getIncrementAllocation() { DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES); return Resources.createResource(incrementMemory, incrementCores); } + + public boolean getAllowUndeclaredPools() { + return getBoolean(ALLOW_UNDECLARED_POOLS, DEFAULT_ALLOW_UNDECLARED_POOLS); + } public boolean getUserAsDefaultQueue() { return getBoolean(USER_AS_DEFAULT_QUEUE, DEFAULT_USER_AS_DEFAULT_QUEUE); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java index 7560309f5e..b891381cd8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java @@ -109,26 +109,27 @@ public void initialize() throws IOException, SAXException, lastSuccessfulReload = scheduler.getClock().getTime(); lastReloadAttempt = scheduler.getClock().getTime(); // Create the default queue - getLeafQueue(YarnConfiguration.DEFAULT_QUEUE_NAME); + getLeafQueue(YarnConfiguration.DEFAULT_QUEUE_NAME, true); } /** - * Get a queue by name, creating it if necessary. If the queue - * is not or can not be a leaf queue, i.e. it already exists as a parent queue, - * or one of the parents in its name is already a leaf queue, null is returned. + * Get a queue by name, creating it if the create param is true and is necessary. + * If the queue is not or can not be a leaf queue, i.e. it already exists as a + * parent queue, or one of the parents in its name is already a leaf queue, + * null is returned. * * The root part of the name is optional, so a queue underneath the root * named "queue1" could be referred to as just "queue1", and a queue named * "queue2" underneath a parent named "parent1" that is underneath the root * could be referred to as just "parent1.queue2". */ - public FSLeafQueue getLeafQueue(String name) { + public FSLeafQueue getLeafQueue(String name, boolean create) { if (!name.startsWith(ROOT_QUEUE + ".")) { name = ROOT_QUEUE + "." + name; } synchronized (queues) { FSQueue queue = queues.get(name); - if (queue == null) { + if (queue == null && create) { FSLeafQueue leafQueue = createLeafQueue(name); if (leafQueue == null) { return null; @@ -223,13 +224,6 @@ public boolean exists(String name) { } } - /** - * Get the queue for a given AppSchedulable. - */ - public FSLeafQueue getQueueForApp(AppSchedulable app) { - return getLeafQueue(app.getApp().getQueueName()); - } - /** * Reload allocations file if it hasn't been loaded in a while */ @@ -384,7 +378,7 @@ public void reloadAllocs() throws IOException, ParserConfigurationException, // Create all queus for (String name: queueNamesInAllocFile) { - getLeafQueue(name); + getLeafQueue(name, true); } // Set custom policies as specified diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 3086afcb5a..e697560b22 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -436,9 +436,9 @@ public void testSimpleHierarchicalFairShareCalculation() { Collection queues = queueManager.getLeafQueues(); assertEquals(3, queues.size()); - FSLeafQueue queue1 = queueManager.getLeafQueue("default"); - FSLeafQueue queue2 = queueManager.getLeafQueue("parent.queue2"); - FSLeafQueue queue3 = queueManager.getLeafQueue("parent.queue3"); + FSLeafQueue queue1 = queueManager.getLeafQueue("default", true); + FSLeafQueue queue2 = queueManager.getLeafQueue("parent.queue2", true); + FSLeafQueue queue3 = queueManager.getLeafQueue("parent.queue3", true); assertEquals(capacity / 2, queue1.getFairShare().getMemory()); assertEquals(capacity / 2, queue1.getMetrics().getFairShareMB()); assertEquals(capacity / 4, queue2.getFairShare().getMemory()); @@ -450,20 +450,20 @@ public void testSimpleHierarchicalFairShareCalculation() { @Test public void testHierarchicalQueuesSimilarParents() { QueueManager queueManager = scheduler.getQueueManager(); - FSLeafQueue leafQueue = queueManager.getLeafQueue("parent.child"); + FSLeafQueue leafQueue = queueManager.getLeafQueue("parent.child", true); Assert.assertEquals(2, queueManager.getLeafQueues().size()); Assert.assertNotNull(leafQueue); Assert.assertEquals("root.parent.child", leafQueue.getName()); - FSLeafQueue leafQueue2 = queueManager.getLeafQueue("parent"); + FSLeafQueue leafQueue2 = queueManager.getLeafQueue("parent", true); Assert.assertNull(leafQueue2); Assert.assertEquals(2, queueManager.getLeafQueues().size()); - FSLeafQueue leafQueue3 = queueManager.getLeafQueue("parent.child.grandchild"); + FSLeafQueue leafQueue3 = queueManager.getLeafQueue("parent.child.grandchild", true); Assert.assertNull(leafQueue3); Assert.assertEquals(2, queueManager.getLeafQueues().size()); - FSLeafQueue leafQueue4 = queueManager.getLeafQueue("parent.sister"); + FSLeafQueue leafQueue4 = queueManager.getLeafQueue("parent.sister", true); Assert.assertNotNull(leafQueue4); Assert.assertEquals("root.parent.sister", leafQueue4.getName()); Assert.assertEquals(3, queueManager.getLeafQueues().size()); @@ -612,9 +612,9 @@ public void testUserAsDefaultQueue() throws Exception { AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent( createAppAttemptId(1, 1), "default", "user1"); scheduler.handle(appAddedEvent); - assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1") + assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true) .getAppSchedulables().size()); - assertEquals(0, scheduler.getQueueManager().getLeafQueue("default") + assertEquals(0, scheduler.getQueueManager().getLeafQueue("default", true) .getAppSchedulables().size()); conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "false"); @@ -622,11 +622,11 @@ public void testUserAsDefaultQueue() throws Exception { AppAddedSchedulerEvent appAddedEvent2 = new AppAddedSchedulerEvent( createAppAttemptId(2, 1), "default", "user2"); scheduler.handle(appAddedEvent2); - assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1") + assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true) .getAppSchedulables().size()); - assertEquals(1, scheduler.getQueueManager().getLeafQueue("default") + assertEquals(1, scheduler.getQueueManager().getLeafQueue("default", true) .getAppSchedulables().size()); - assertEquals(0, scheduler.getQueueManager().getLeafQueue("user2") + assertEquals(0, scheduler.getQueueManager().getLeafQueue("user2", true) .getAppSchedulables().size()); } @@ -772,7 +772,7 @@ public void testAppAdditionAndRemoval() throws Exception { assertEquals(2, scheduler.getQueueManager().getLeafQueues().size()); // That queue should have one app - assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1") + assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true) .getAppSchedulables().size()); AppRemovedSchedulerEvent appRemovedEvent1 = new AppRemovedSchedulerEvent( @@ -782,7 +782,7 @@ public void testAppAdditionAndRemoval() throws Exception { scheduler.handle(appRemovedEvent1); // Queue should have no apps - assertEquals(0, scheduler.getQueueManager().getLeafQueue("user1") + assertEquals(0, scheduler.getQueueManager().getLeafQueue("user1", true) .getAppSchedulables().size()); } @@ -919,10 +919,10 @@ public void testHierarchicalQueueAllocationFileParsing() throws IOException, SAX Collection leafQueues = queueManager.getLeafQueues(); Assert.assertEquals(4, leafQueues.size()); - Assert.assertNotNull(queueManager.getLeafQueue("queueA")); - Assert.assertNotNull(queueManager.getLeafQueue("queueB.queueC")); - Assert.assertNotNull(queueManager.getLeafQueue("queueB.queueD")); - Assert.assertNotNull(queueManager.getLeafQueue("default")); + Assert.assertNotNull(queueManager.getLeafQueue("queueA", true)); + Assert.assertNotNull(queueManager.getLeafQueue("queueB.queueC", true)); + Assert.assertNotNull(queueManager.getLeafQueue("queueB.queueD", true)); + Assert.assertNotNull(queueManager.getLeafQueue("default", true)); // Make sure querying for queues didn't create any new ones: Assert.assertEquals(4, leafQueues.size()); } @@ -1423,9 +1423,9 @@ public void testPreemptionDecision() throws Exception { scheduler.update(); FSLeafQueue schedC = - scheduler.getQueueManager().getLeafQueue("queueC"); + scheduler.getQueueManager().getLeafQueue("queueC", true); FSLeafQueue schedD = - scheduler.getQueueManager().getLeafQueue("queueD"); + scheduler.getQueueManager().getLeafQueue("queueD", true); assertTrue(Resources.equals( Resources.none(), scheduler.resToPreempt(schedC, clock.getTime()))); @@ -1688,7 +1688,7 @@ public void testFifoWithinQueue() throws Exception { FSSchedulerApp app1 = scheduler.applications.get(attId1); FSSchedulerApp app2 = scheduler.applications.get(attId2); - FSLeafQueue queue1 = scheduler.getQueueManager().getLeafQueue("queue1"); + FSLeafQueue queue1 = scheduler.getQueueManager().getLeafQueue("queue1", true); queue1.setPolicy(new FifoPolicy()); scheduler.update(); @@ -1716,7 +1716,7 @@ public void testFifoWithinQueue() throws Exception { public void testMaxAssign() throws AllocationConfigurationException { // set required scheduler configs scheduler.assignMultiple = true; - scheduler.getQueueManager().getLeafQueue("root.default") + scheduler.getQueueManager().getLeafQueue("root.default", true) .setPolicy(SchedulingPolicy.getDefault()); RMNode node = @@ -1793,7 +1793,7 @@ public void testAssignContainer() throws Exception { FSSchedulerApp app3 = scheduler.applications.get(attId3); FSSchedulerApp app4 = scheduler.applications.get(attId4); - scheduler.getQueueManager().getLeafQueue(fifoQueue) + scheduler.getQueueManager().getLeafQueue(fifoQueue, true) .setPolicy(SchedulingPolicy.parse("fifo")); scheduler.update(); @@ -2405,4 +2405,46 @@ private void handleExternalEvent(SchedulerEvent event) throws Exception { //expected } } + + @Test + public void testDontAllowUndeclaredPools() throws Exception{ + Configuration conf = createConfiguration(); + conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.close(); + + QueueManager queueManager = scheduler.getQueueManager(); + queueManager.initialize(); + + FSLeafQueue jerryQueue = queueManager.getLeafQueue("jerry", false); + FSLeafQueue defaultQueue = queueManager.getLeafQueue("default", false); + + // Should get put into jerry + createSchedulingRequest(1024, "jerry", "someuser"); + assertEquals(1, jerryQueue.getAppSchedulables().size()); + + // Should get forced into default + createSchedulingRequest(1024, "newqueue", "someuser"); + assertEquals(1, jerryQueue.getAppSchedulables().size()); + assertEquals(1, defaultQueue.getAppSchedulables().size()); + + // Would get put into someuser because of user-as-default-queue, but should + // be forced into default + createSchedulingRequest(1024, "default", "someuser"); + assertEquals(1, jerryQueue.getAppSchedulables().size()); + assertEquals(2, defaultQueue.getAppSchedulables().size()); + + // Should get put into jerry because of user-as-default-queue + createSchedulingRequest(1024, "default", "jerry"); + assertEquals(2, jerryQueue.getAppSchedulables().size()); + assertEquals(2, defaultQueue.getAppSchedulables().size()); + } }