From 55fcbcb5c2a096f98f273fda52ae25ecaa1d8bb6 Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Wed, 15 Apr 2020 07:24:04 +0200 Subject: [PATCH] YARN-10234. FS-CS converter: don't enable auto-create queue property for root. Contributed by Peter Bacsko --- .../fair/converter/FSQueueConverter.java | 4 +++- .../TestFSConfigToCSConfigConverter.java | 23 +++++++++++++++---- .../fair/converter/TestFSQueueConverter.java | 2 +- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java index a84b6cbe0d..2b3f28495d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java @@ -27,6 +27,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.ConfigurableResource; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; @@ -225,7 +226,8 @@ private void emitPreemptionDisabled(String queueName, FSQueue queue) { * @param queueName */ private void emitAutoCreateChildQueue(String queueName, FSQueue queue) { - if (autoCreateChildQueues && !queue.getChildQueues().isEmpty()) { + if (autoCreateChildQueues && !queue.getChildQueues().isEmpty() + && !queueName.equals(CapacitySchedulerConfiguration.ROOT)) { capacitySchedulerConfig.setBoolean(PREFIX + queueName + ".auto-create-child-queue.enabled", true); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java index 4ceb8c8572..dd3e6d4588 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java @@ -649,6 +649,7 @@ public void testAutoCreateChildQueuesFalseWithoutPlacementRules() testAutoCreateChildQueuesWithoutPlacementRules(false); } + @SuppressWarnings("checkstyle:linelength") private void testAutoCreateChildQueuesWithoutPlacementRules( boolean allowUndeclaredPools) throws Exception { config = new Configuration(false); @@ -661,15 +662,27 @@ private void testAutoCreateChildQueuesWithoutPlacementRules( converter.convert(config); Configuration convertedConf = converter.getCapacitySchedulerConfig(); - String property = + String rootUserAutoCreate = + "yarn.scheduler.capacity.root.users.auto-create-child-queue.enabled"; + String rootAutoCreate = "yarn.scheduler.capacity.root.auto-create-child-queue.enabled"; + String leafQueueAutoCreate = + "yarn.scheduler.capacity.root.users.joe.auto-create-child-queue.enabled"; if (allowUndeclaredPools) { - assertEquals("Auto-create queue wasn't enabled", true, - convertedConf.getBoolean(property, false)); + assertEquals("Auto-create queue wasn't enabled for root.users", true, + convertedConf.getBoolean(rootUserAutoCreate, false)); + assertNull("Auto-create queue shouldn't be set for root", + convertedConf.get(rootAutoCreate)); + assertNull("Auto-create queue shouldn't be set for leaf", + convertedConf.get(leafQueueAutoCreate)); } else { - assertNull("Auto-create queue shouldn't be set", - convertedConf.get(property)); + assertNull("Auto-create queue shouldn't be set for root.users", + convertedConf.get(rootUserAutoCreate)); + assertNull("Auto-create queue shouldn't be set for root", + convertedConf.get(rootAutoCreate)); + assertNull("Auto-create queue shouldn't be set for leaf", + convertedConf.get(leafQueueAutoCreate)); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java index 41018623e8..0ed6bbf202 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java @@ -316,7 +316,7 @@ public void testQueueAutoCreateChildQueue() { converter.convertQueueHierarchy(rootQueue); - Set parentQueues = Sets.newHashSet("root", + Set parentQueues = Sets.newHashSet( "root.admins", "root.users");