From 6c56612af54152d5f96acbdf1aa7f7954d1e6eae Mon Sep 17 00:00:00 2001 From: Sanford Ryza Date: Fri, 23 May 2014 22:38:52 +0000 Subject: [PATCH] YARN-2012. Fair Scheduler: allow default queue placement rule to take an arbitrary queue (Ashwin Shankar via Sandy Ryza) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1597204 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../scheduler/fair/QueuePlacementRule.java | 25 ++++++++++--- .../fair/TestQueuePlacementPolicy.java | 35 +++++++++++++++++++ .../src/site/apt/FairScheduler.apt.vm | 5 +-- 4 files changed, 62 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 76d5ecce89..8bc5f4ea49 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -99,6 +99,9 @@ Release 2.5.0 - UNRELEASED YARN-1937. Added owner-only ACLs support for Timeline Client and server. (Zhijie Shen via vinodkv) + YARN-2012. Fair Scheduler: allow default queue placement rule to take an + arbitrary queue (Ashwin Shankar via Sandy Ryza) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java index fa50bb7d30..a16e35a3bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java @@ -276,21 +276,38 @@ public boolean isTerminal() { } /** - * Places all apps in the default queue + * Places apps in the specified default queue. If no default queue is + * specified the app is placed in root.default queue. */ public static class Default extends QueuePlacementRule { + private String defaultQueueName; + + @Override + public void initializeFromXml(Element el) + throws AllocationConfigurationException { + defaultQueueName = el.getAttribute("queue"); + if (defaultQueueName != null && !defaultQueueName.isEmpty()) { + if (!defaultQueueName.startsWith("root.")) { + defaultQueueName = "root." + defaultQueueName; + } + } else { + defaultQueueName = "root." + YarnConfiguration.DEFAULT_QUEUE_NAME; + } + super.initializeFromXml(el); + } + @Override protected String getQueueForApp(String requestedQueue, String user, Groups groups, Map> configuredQueues) { - return "root." + YarnConfiguration.DEFAULT_QUEUE_NAME; + return defaultQueueName; } - + @Override public boolean isTerminal() { return true; } } - + /** * Rejects all apps */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java index 640d771f2c..e20b0c3307 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java @@ -133,6 +133,22 @@ public void testTerminals() throws Exception { parse(sb.toString()); } + @Test + public void testDefaultRuleWithQueueAttribute() throws Exception { + // This test covers the use case where we would like default rule + // to point to a different queue by default rather than root.default + configuredQueues.get(FSQueueType.LEAF).add("root.someDefaultQueue"); + StringBuffer sb = new StringBuffer(); + sb.append(""); + sb.append(" "); + sb.append(" "); + sb.append(""); + + QueuePlacementPolicy policy = parse(sb.toString()); + assertEquals("root.someDefaultQueue", + policy.assignAppToQueue("root.default", "user1")); + } + @Test public void testNestedUserQueueParsingErrors() { // No nested rule specified in hierarchical user queue @@ -311,6 +327,25 @@ public void testNestedUserQueueSpecificRule() throws Exception { policy.assignAppToQueue("root.parent2", "user2")); } + @Test + public void testNestedUserQueueDefaultRule() throws Exception { + // This test covers the use case where we would like user queues to be + // created under a default parent queue + configuredQueues.get(FSQueueType.PARENT).add("root.parentq"); + StringBuffer sb = new StringBuffer(); + sb.append(""); + sb.append(" "); + sb.append(" "); + sb.append(" "); + sb.append(" "); + sb.append(" "); + sb.append(""); + + QueuePlacementPolicy policy = parse(sb.toString()); + assertEquals("root.parentq.user1", + policy.assignAppToQueue("root.default", "user1")); + } + private QueuePlacementPolicy parse(String str) throws Exception { // Read and parse the allocations file. DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm index 02890a1c35..611c390a82 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm @@ -310,7 +310,8 @@ Allocation file format to ‘parent’ or by configuring at least one leaf under that queue which makes it a parent. See example allocation for a sample use case. - * default: the app is placed into the queue named "default". + * default: the app is placed into the queue specified in the ‘queue’ attribute of the + default rule. If ‘queue’ attribute is not specified, the app is placed into ‘root.default’ queue. * reject: the app is rejected. @@ -348,7 +349,7 @@ Allocation file format - + ---