From 743a5b78e3bce26482c6bcd71208e422fa178cf9 Mon Sep 17 00:00:00 2001 From: Arun Murthy Date: Thu, 29 Dec 2011 19:06:34 +0000 Subject: [PATCH] MAPREDUCE-3522. Ensure queues inherit ACLs from parent if they aren't explicitly specified. Contributed by Jonathan Eagles. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1225618 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 ++ .../scheduler/capacity/CapacityScheduler.java | 9 ++---- .../CapacitySchedulerConfiguration.java | 29 ++++++++++++------- .../capacity/TestApplicationLimits.java | 10 +++---- .../capacity/TestCapacityScheduler.java | 14 ++++----- .../scheduler/capacity/TestLeafQueue.java | 28 ++++++++++++------ .../scheduler/capacity/TestParentQueue.java | 24 +++++++-------- .../scheduler/capacity/TestQueueParsing.java | 20 ++++++------- .../resourcemanager/webapp/TestRMWebApp.java | 10 +++---- .../src/site/apt/CapacityScheduler.apt.vm | 12 +++++--- 10 files changed, 91 insertions(+), 68 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index e1c1771509..2529cacfd3 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -378,6 +378,9 @@ Release 0.23.1 - Unreleased MAPREDUCE-3521. Fixed streaming to ensure it doesn't silently ignore unknown arguments. (Robert Evans via acmurthy) + MAPREDUCE-3522. Ensure queues inherit ACLs from parent if they aren't + explicitly specified. (Jonathan Eagles via acmurthy) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 8aa85c3ab1..a542d0339c 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -200,12 +200,9 @@ public synchronized void reinitialize(Configuration conf, } } - @Private - public static final String ROOT = "root"; - @Private public static final String ROOT_QUEUE = - CapacitySchedulerConfiguration.PREFIX + ROOT; + CapacitySchedulerConfiguration.PREFIX + CapacitySchedulerConfiguration.ROOT; static class QueueHook { public CSQueue hook(CSQueue queue) { @@ -218,7 +215,7 @@ public CSQueue hook(CSQueue queue) { private void initializeQueues(CapacitySchedulerConfiguration conf) throws IOException { root = - parseQueue(this, conf, null, ROOT, queues, queues, + parseQueue(this, conf, null, CapacitySchedulerConfiguration.ROOT, queues, queues, queueComparator, applicationComparator, noop); LOG.info("Initialized root queue " + root); } @@ -229,7 +226,7 @@ private void reinitializeQueues(CapacitySchedulerConfiguration conf) // Parse new queues Map newQueues = new HashMap(); CSQueue newRoot = - parseQueue(this, conf, null, ROOT, newQueues, queues, + parseQueue(this, conf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, queueComparator, applicationComparator, noop); // Ensure all existing queues are still present diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 82f6f85418..911b9ef673 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -104,14 +104,20 @@ public class CapacitySchedulerConfiguration extends Configuration { @Private public static final float DEFAULT_USER_LIMIT_FACTOR = 1.0f; - + @Private - public static final String DEFAULT_ACL = "*"; + public static final String ALL_ACL = "*"; + + @Private + public static final String NONE_ACL = " "; @Private public static final String ENABLE_USER_METRICS = PREFIX +"user-metrics.enable"; @Private public static final boolean DEFAULT_ENABLE_USER_METRICS = false; + @Private + public static final String ROOT = "root"; + public CapacitySchedulerConfiguration() { this(new Configuration()); } @@ -143,14 +149,14 @@ public int getCapacity(String queue) { throw new IllegalArgumentException("Illegal " + "capacity of " + capacity + " for queue " + queue); } - LOG.info("CSConf - setCapacity: queuePrefix=" + getQueuePrefix(queue) + + LOG.debug("CSConf - setCapacity: queuePrefix=" + getQueuePrefix(queue) + ", capacity=" + capacity); return capacity; } public void setCapacity(String queue, int capacity) { setInt(getQueuePrefix(queue) + CAPACITY, capacity); - LOG.info("CSConf - setCapacity: queuePrefix=" + getQueuePrefix(queue) + + LOG.debug("CSConf - setCapacity: queuePrefix=" + getQueuePrefix(queue) + ", capacity=" + capacity); } @@ -162,7 +168,7 @@ public int getMaximumCapacity(String queue) { public void setMaximumCapacity(String queue, int maxCapacity) { setInt(getQueuePrefix(queue) + MAXIMUM_CAPACITY, maxCapacity); - LOG.info("CSConf - setMaxCapacity: queuePrefix=" + getQueuePrefix(queue) + + LOG.debug("CSConf - setMaxCapacity: queuePrefix=" + getQueuePrefix(queue) + ", maxCapacity=" + maxCapacity); } @@ -174,7 +180,7 @@ public int getUserLimit(String queue) { public void setUserLimit(String queue, int userLimit) { setInt(getQueuePrefix(queue) + USER_LIMIT, userLimit); - LOG.info("here setUserLimit: queuePrefix=" + getQueuePrefix(queue) + + LOG.debug("here setUserLimit: queuePrefix=" + getQueuePrefix(queue) + ", userLimit=" + getUserLimit(queue)); } @@ -201,7 +207,10 @@ private static String getAclKey(QueueACL acl) { public AccessControlList getAcl(String queue, QueueACL acl) { String queuePrefix = getQueuePrefix(queue); - String aclString = get(queuePrefix + getAclKey(acl), DEFAULT_ACL); + // The root queue defaults to all access if not defined + // Sub queues inherit access if not defined + String defaultAcl = queue.equals(ROOT) ? ALL_ACL : NONE_ACL; + String aclString = get(queuePrefix + getAclKey(acl), defaultAcl); return new AccessControlList(aclString); } @@ -226,16 +235,16 @@ public void setAcls(String queue, Map acls) { } public String[] getQueues(String queue) { - LOG.info("CSConf - getQueues called for: queuePrefix=" + getQueuePrefix(queue)); + LOG.debug("CSConf - getQueues called for: queuePrefix=" + getQueuePrefix(queue)); String[] queues = getStrings(getQueuePrefix(queue) + QUEUES); - LOG.info("CSConf - getQueues: queuePrefix=" + getQueuePrefix(queue) + + LOG.debug("CSConf - getQueues: queuePrefix=" + getQueuePrefix(queue) + ", queues=" + ((queues == null) ? "" : StringUtils.arrayToString(queues))); return queues; } public void setQueues(String queue, String[] subQueues) { set(getQueuePrefix(queue) + QUEUES, StringUtils.arrayToString(subQueues)); - LOG.info("CSConf - setQueues: qPrefix=" + getQueuePrefix(queue) + + LOG.debug("CSConf - setQueues: qPrefix=" + getQueuePrefix(queue) + ", queues=" + StringUtils.arrayToString(subQueues)); } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index 3db772c498..824c8086a4 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -88,13 +88,13 @@ public void setUp() throws IOException { private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues(CapacityScheduler.ROOT, new String[] {A, B}); - conf.setCapacity(CapacityScheduler.ROOT, 100); + conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B}); + conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100); - final String Q_A = CapacityScheduler.ROOT + "." + A; + final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A; conf.setCapacity(Q_A, 10); - final String Q_B = CapacityScheduler.ROOT + "." + B; + final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B; conf.setCapacity(Q_B, 90); LOG.info("Setup top-level queues a and b"); @@ -252,7 +252,7 @@ public void testActiveApplicationLimits() throws Exception { public void testHeadroom() throws Exception { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); - csConf.setUserLimit(CapacityScheduler.ROOT + "." + A, 25); + csConf.setUserLimit(CapacitySchedulerConfiguration.ROOT + "." + A, 25); setupQueueConfiguration(csConf); CapacitySchedulerContext csContext = mock(CapacitySchedulerContext.class); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index a43ea51c3f..57d62aaa46 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -197,13 +197,13 @@ public void testCapacityScheduler() throws Exception { private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues(CapacityScheduler.ROOT, new String[] {"a", "b"}); - conf.setCapacity(CapacityScheduler.ROOT, 100); + conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"}); + conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100); - final String A = CapacityScheduler.ROOT + ".a"; + final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setCapacity(A, 10); - final String B = CapacityScheduler.ROOT + ".b"; + final String B = CapacitySchedulerConfiguration.ROOT + ".b"; conf.setCapacity(B, 90); // Define 2nd-level queues @@ -250,9 +250,9 @@ public void testParseQueue() throws IOException { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(conf); - conf.setQueues(CapacityScheduler.ROOT + ".a.a1", new String[] {"b1"} ); - conf.setCapacity(CapacityScheduler.ROOT + ".a.a1.b1", 100); - conf.setUserLimitFactor(CapacityScheduler.ROOT + ".a.a1.b1", 100.0f); + conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a.a1", new String[] {"b1"} ); + conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100); + conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f); cs.reinitialize(conf, null, null); } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 69fea5d998..96dbe3b5a3 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -107,29 +107,35 @@ public void setUp() throws Exception { private static final String A = "a"; private static final String B = "b"; private static final String C = "c"; + private static final String C1 = "c1"; private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues(CapacityScheduler.ROOT, new String[] {A, B, C}); - conf.setCapacity(CapacityScheduler.ROOT, 100); - conf.setMaximumCapacity(CapacityScheduler.ROOT, 100); - conf.setAcl(CapacityScheduler.ROOT, QueueACL.SUBMIT_APPLICATIONS, " "); + conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B, C}); + conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100); + conf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT, 100); + conf.setAcl(CapacitySchedulerConfiguration.ROOT, QueueACL.SUBMIT_APPLICATIONS, " "); - final String Q_A = CapacityScheduler.ROOT + "." + A; + final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A; conf.setCapacity(Q_A, 9); conf.setMaximumCapacity(Q_A, 20); conf.setAcl(Q_A, QueueACL.SUBMIT_APPLICATIONS, "*"); - final String Q_B = CapacityScheduler.ROOT + "." + B; + final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B; conf.setCapacity(Q_B, 90); conf.setMaximumCapacity(Q_B, 99); conf.setAcl(Q_B, QueueACL.SUBMIT_APPLICATIONS, "*"); - final String Q_C = CapacityScheduler.ROOT + "." + C; + final String Q_C = CapacitySchedulerConfiguration.ROOT + "." + C; conf.setCapacity(Q_C, 1); conf.setMaximumCapacity(Q_C, 10); conf.setAcl(Q_C, QueueACL.SUBMIT_APPLICATIONS, " "); + conf.setQueues(Q_C, new String[] {C1}); + + final String Q_C1 = Q_C + "." + C1; + conf.setCapacity(Q_C1, 100); + LOG.info("Setup top-level queues a and b"); } @@ -191,7 +197,7 @@ public void testInitializeQueue() throws Exception { assertEquals(0.99, b.getMaximumCapacity(), epsilon); assertEquals(0.99, b.getAbsoluteMaximumCapacity(), epsilon); - LeafQueue c = stubLeafQueue((LeafQueue)queues.get(C)); + ParentQueue c = (ParentQueue)queues.get(C); assertEquals(0.01, c.getCapacity(), epsilon); assertEquals(0.01, c.getAbsoluteCapacity(), epsilon); assertEquals(0.1, c.getMaximumCapacity(), epsilon); @@ -1117,12 +1123,14 @@ public void testInheritedQueueAcls() throws IOException { LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A)); LeafQueue b = stubLeafQueue((LeafQueue)queues.get(B)); - LeafQueue c = stubLeafQueue((LeafQueue)queues.get(C)); + ParentQueue c = (ParentQueue)queues.get(C); + LeafQueue c1 = stubLeafQueue((LeafQueue)queues.get(C1)); assertFalse(root.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user)); assertTrue(a.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user)); assertTrue(b.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user)); assertFalse(c.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user)); + assertFalse(c1.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user)); assertTrue(hasQueueACL( a.getQueueUserAclInfo(user), QueueACL.SUBMIT_APPLICATIONS)); @@ -1130,6 +1138,8 @@ public void testInheritedQueueAcls() throws IOException { b.getQueueUserAclInfo(user), QueueACL.SUBMIT_APPLICATIONS)); assertFalse(hasQueueACL( c.getQueueUserAclInfo(user), QueueACL.SUBMIT_APPLICATIONS)); + assertFalse(hasQueueACL( + c1.getQueueUserAclInfo(user), QueueACL.SUBMIT_APPLICATIONS)); } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java index 791d5de1ab..638ed0edb8 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java @@ -70,13 +70,13 @@ public void setUp() throws Exception { private void setupSingleLevelQueues(CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues(CapacityScheduler.ROOT, new String[] {A, B}); - conf.setCapacity(CapacityScheduler.ROOT, 100); + conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B}); + conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100); - final String Q_A = CapacityScheduler.ROOT + "." + A; + final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A; conf.setCapacity(Q_A, 30); - final String Q_B = CapacityScheduler.ROOT + "." + B; + final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B; conf.setCapacity(Q_B, 70); LOG.info("Setup top-level queues a and b"); @@ -144,7 +144,7 @@ public void testSingleLevelQueues() throws Exception { Map queues = new HashMap(); CSQueue root = CapacityScheduler.parseQueue(csContext, csConf, null, - CapacityScheduler.ROOT, queues, queues, + CapacitySchedulerConfiguration.ROOT, queues, queues, CapacityScheduler.queueComparator, CapacityScheduler.applicationComparator, TestUtils.spyHook); @@ -246,19 +246,19 @@ public void testSingleLevelQueues() throws Exception { private void setupMultiLevelQueues(CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues(CapacityScheduler.ROOT, new String[] {A, B, C, D}); - conf.setCapacity(CapacityScheduler.ROOT, 100); + conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B, C, D}); + conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100); - final String Q_A = CapacityScheduler.ROOT + "." + A; + final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A; conf.setCapacity(Q_A, 10); - final String Q_B = CapacityScheduler.ROOT + "." + B; + final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B; conf.setCapacity(Q_B, 50); - final String Q_C = CapacityScheduler.ROOT + "." + C; + final String Q_C = CapacitySchedulerConfiguration.ROOT + "." + C; conf.setCapacity(Q_C, 20); - final String Q_D = CapacityScheduler.ROOT + "." + D; + final String Q_D = CapacitySchedulerConfiguration.ROOT + "." + D; conf.setCapacity(Q_D, 20); // Define 2-nd level queues @@ -282,7 +282,7 @@ public void testMultiLevelQueues() throws Exception { Map queues = new HashMap(); CSQueue root = CapacityScheduler.parseQueue(csContext, csConf, null, - CapacityScheduler.ROOT, queues, queues, + CapacitySchedulerConfiguration.ROOT, queues, queues, CapacityScheduler.queueComparator, CapacityScheduler.applicationComparator, TestUtils.spyHook); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java index 28a0026cec..c78924644e 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java @@ -42,16 +42,16 @@ public void testQueueParsing() throws Exception { private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues(CapacityScheduler.ROOT, new String[] {"a", "b", "c"}); - conf.setCapacity(CapacityScheduler.ROOT, 100); + conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"}); + conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100); - final String A = CapacityScheduler.ROOT + ".a"; + final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setCapacity(A, 10); - final String B = CapacityScheduler.ROOT + ".b"; + final String B = CapacitySchedulerConfiguration.ROOT + ".b"; conf.setCapacity(B, 20); - final String C = CapacityScheduler.ROOT + ".c"; + final String C = CapacitySchedulerConfiguration.ROOT + ".c"; conf.setCapacity(C, 70); LOG.info("Setup top-level queues"); @@ -100,7 +100,7 @@ public void testRootQueueParsing() throws Exception { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); // non-100 percent value will throw IllegalArgumentException - conf.setCapacity(CapacityScheduler.ROOT, 90); + conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 90); CapacityScheduler capacityScheduler = new CapacityScheduler(); capacityScheduler.reinitialize(conf, null, null); @@ -109,14 +109,14 @@ public void testRootQueueParsing() throws Exception { public void testMaxCapacity() throws Exception { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); - conf.setQueues(CapacityScheduler.ROOT, new String[] {"a", "b", "c"}); - conf.setCapacity(CapacityScheduler.ROOT, 100); + conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"}); + conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100); - final String A = CapacityScheduler.ROOT + ".a"; + final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setCapacity(A, 50); conf.setMaximumCapacity(A, 60); - final String B = CapacityScheduler.ROOT + ".b"; + final String B = CapacitySchedulerConfiguration.ROOT + ".b"; conf.setCapacity(B, 50); conf.setMaximumCapacity(B, 45); // Should throw an exception diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java index f567419545..0acbd1f42a 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java @@ -155,16 +155,16 @@ public static CapacityScheduler mockCapacityScheduler() throws IOException { static void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues(CapacityScheduler.ROOT, new String[] {"a", "b", "c"}); - conf.setCapacity(CapacityScheduler.ROOT, 100); + conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"}); + conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100); - final String A = CapacityScheduler.ROOT + ".a"; + final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setCapacity(A, 10); - final String B = CapacityScheduler.ROOT + ".b"; + final String B = CapacitySchedulerConfiguration.ROOT + ".b"; conf.setCapacity(B, 20); - final String C = CapacityScheduler.ROOT + ".c"; + final String C = CapacitySchedulerConfiguration.ROOT + ".c"; conf.setCapacity(C, 70); // Define 2nd-level queues diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm index 88fb9a6c88..0ee8220425 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm @@ -161,7 +161,7 @@ Hadoop MapReduce Next Generation - Capacity Scheduler A given queue's children can be defined with the configuration knob: <<.queues>>>. Children do not - inherit properties directly from the parent. + inherit properties directly from the parent unless otherwise noted. Here is an example with three top-level child-queues <<>>, <<>> and <<>> and some sub-queues for <<>> and <<>>: @@ -274,17 +274,21 @@ Hadoop MapReduce Next Generation - Capacity Scheduler | <<.acl_submit_jobs>>> | | | | The which controls who can jobs to the given queue. | | | If the given user/group has necessary ACLs on the given queue or | -| | they can submit jobs. | +| | they can submit jobs. | +| | for this property inherited from the parent queue | +| | if not specified. | *--------------------------------------+--------------------------------------+ | <<.acl_administer_jobs>>> | | | | The which controls who can jobs on the given queue. | | | If the given user/group has necessary ACLs on the given queue or | -| | they can administer jobs. | +| | they can administer jobs. | +| | for this property inherited from the parent queue | +| | if not specified. | *--------------------------------------+--------------------------------------+ An is of the form , , . The special value of <<*>> implies . The special value of - implies . The default is <<*>> if not specified. + implies . The default is <<*>> for the root queue if not specified. * Reviewing the configuration of the CapacityScheduler