From f85b74ccf9f1c1c1444cc00750b03468cbf40fb9 Mon Sep 17 00:00:00 2001 From: Daniel Templeton Date: Thu, 26 Jan 2017 10:31:09 -0800 Subject: [PATCH] YARN-4975. Fair Scheduler: exception thrown when a parent queue marked 'parent' has configured child queues (Contributed by Yufei Gu via Daniel Templeton) --- .../fair/AllocationFileLoaderService.java | 26 +++--- .../fair/TestAllocationFileLoaderService.java | 88 +++++++++++++++++++ 2 files changed, 101 insertions(+), 13 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java index cd4a19b573..163a265918 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java @@ -487,6 +487,7 @@ private void loadQueue(String parentName, Element element, Map racls = new HashMap<>(); NodeList fields = element.getChildNodes(); boolean isLeaf = true; + boolean isReservable = false; for (int j = 0; j < fields.getLength(); j++) { Node fieldNode = fields.item(j); @@ -558,7 +559,7 @@ private void loadQueue(String parentName, Element element, racls.put(ReservationACL.SUBMIT_RESERVATIONS, new AccessControlList(text)); } else if ("reservation".equals(field.getTagName())) { - isLeaf = false; + isReservable = true; reservableQueues.add(queueName); configuredQueues.get(FSQueueType.PARENT).add(queueName); } else if ("allowPreemptionFrom".equals(field.getTagName())) { @@ -577,22 +578,21 @@ private void loadQueue(String parentName, Element element, isLeaf = false; } } - if (isLeaf) { - // if a leaf in the alloc file is marked as type='parent' - // then store it under 'parent' - if ("parent".equals(element.getAttribute("type"))) { - configuredQueues.get(FSQueueType.PARENT).add(queueName); - } else { - configuredQueues.get(FSQueueType.LEAF).add(queueName); - } + + // if a leaf in the alloc file is marked as type='parent' + // then store it as a parent queue + if (isLeaf && !"parent".equals(element.getAttribute("type"))) { + configuredQueues.get(FSQueueType.LEAF).add(queueName); } else { - if ("parent".equals(element.getAttribute("type"))) { - throw new AllocationConfigurationException("Both and " + - "type=\"parent\" found for queue " + queueName + " which is " + - "unsupported"); + if (isReservable) { + throw new AllocationConfigurationException("The configuration settings" + + " for " + queueName + " are invalid. A queue element that " + + "contains child queue elements or that has the type='parent' " + + "attribute cannot also include a reservation element."); } configuredQueues.get(FSQueueType.PARENT).add(queueName); } + // Set default acls if not defined // The root queue defaults to all access for (QueueACL acl : QueueACL.values()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java index 12c3fa9cc9..c8b9ad8bde 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java @@ -618,6 +618,94 @@ public void testQueueNameContainingOnlyWhitespace() throws Exception { allocLoader.reloadAllocations(); } + @Test + public void testParentTagWithReservation() throws Exception { + Configuration conf = new Configuration(); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.close(); + + AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); + allocLoader.init(conf); + ReloadListener confHolder = new ReloadListener(); + allocLoader.setReloadListener(confHolder); + try { + allocLoader.reloadAllocations(); + } catch (AllocationConfigurationException ex) { + assertEquals(ex.getMessage(), "The configuration settings for root.parent" + + " are invalid. A queue element that contains child queue elements" + + " or that has the type='parent' attribute cannot also include a" + + " reservation element."); + } + } + + @Test + public void testParentWithReservation() throws Exception { + Configuration conf = new Configuration(); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println(" "); + out.println(" "); + out.println(""); + out.println(""); + out.close(); + + AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); + allocLoader.init(conf); + ReloadListener confHolder = new ReloadListener(); + allocLoader.setReloadListener(confHolder); + try { + allocLoader.reloadAllocations(); + } catch (AllocationConfigurationException ex) { + assertEquals(ex.getMessage(), "The configuration settings for root.parent" + + " are invalid. A queue element that contains child queue elements" + + " or that has the type='parent' attribute cannot also include a" + + " reservation element."); + } + } + + @Test + public void testParentTagWithChild() throws Exception { + Configuration conf = new Configuration(); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println(" "); + out.println(" "); + out.println(""); + out.println(""); + out.close(); + + AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); + allocLoader.init(conf); + ReloadListener confHolder = new ReloadListener(); + allocLoader.setReloadListener(confHolder); + allocLoader.reloadAllocations(); + AllocationConfiguration queueConf = confHolder.allocConf; + // Check whether queue 'parent' and 'child' are loaded successfully + assertTrue(queueConf.getConfiguredQueues().get(FSQueueType.PARENT) + .contains("root.parent")); + assertTrue(queueConf.getConfiguredQueues().get(FSQueueType.LEAF) + .contains("root.parent.child")); + } + /** * Verify that you can't have the queue name with just a non breaking * whitespace in the allocations file.