From cfe6e1f7da95d2ae05e55e5244925d296df4bbd4 Mon Sep 17 00:00:00 2001 From: Szilard Nemeth Date: Wed, 20 Jan 2021 15:22:44 +0100 Subject: [PATCH] YARN-10578. Fix Auto Queue Creation parent handling. Contributed by Andras Gyori --- .../scheduler/capacity/AbstractCSQueue.java | 19 -------------- .../scheduler/capacity/CapacityScheduler.java | 25 +++++++++---------- .../CapacitySchedulerAutoQueueHandler.java | 5 +--- ...CapacitySchedulerNewQueueAutoCreation.java | 3 +-- 4 files changed, 14 insertions(+), 38 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index c02e1d3ea1..bc3ff2294f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -154,11 +154,6 @@ public enum CapacityConfigType { // is it a dynamic queue? private boolean dynamicQueue = false; - // When this queue has application submit to? - // This property only applies to dynamic queue, - // and will be used to check when the queue need to be removed. - private long lastSubmittedTimestamp; - public AbstractCSQueue(CapacitySchedulerContext cs, String queueName, CSQueue parent, CSQueue old) throws IOException { this(cs, cs.getConfiguration(), queueName, parent, old); @@ -1633,18 +1628,4 @@ public void setDynamicQueue(boolean dynamicQueue) { writeLock.unlock(); } } - - public long getLastSubmittedTimestamp() { - return lastSubmittedTimestamp; - } - - // "Tab" the queue, so this queue won't be removed because of idle timeout. - public void signalToSubmitToQueue() { - writeLock.lock(); - try { - this.lastSubmittedTimestamp = System.currentTimeMillis(); - } finally { - writeLock.unlock(); - } - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 5ee557a689..204fa73b45 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -343,7 +343,7 @@ void initScheduler(Configuration configuration) throws this.queueManager.setCapacitySchedulerContext(this); this.autoQueueHandler = new CapacitySchedulerAutoQueueHandler( - this.queueManager, this.conf); + this.queueManager); this.workflowPriorityMappingsMgr = new WorkflowPriorityMappingsManager(); @@ -3380,26 +3380,25 @@ private LeafQueue autoCreateLeafQueue( if (!StringUtils.isEmpty(parentQueueName)) { CSQueue parentQueue = getQueue(parentQueueName); - if (parentQueue == null) { - throw new SchedulerDynamicEditException( - "Could not auto-create leaf queue for " + leafQueueName - + ". Queue mapping specifies an invalid parent queue " - + "which does not exist " + parentQueueName); - } - - if (conf.isAutoCreateChildQueueEnabled(parentQueue.getQueuePath())) { + if (parentQueue != null && + conf.isAutoCreateChildQueueEnabled(parentQueue.getQueuePath())) { // Case 1: Handle ManagedParentQueue - AutoCreatedLeafQueue autoCreatedLeafQueue = null; ManagedParentQueue autoCreateEnabledParentQueue = (ManagedParentQueue) parentQueue; - autoCreatedLeafQueue = new AutoCreatedLeafQueue(this, leafQueueName, - autoCreateEnabledParentQueue); + AutoCreatedLeafQueue autoCreatedLeafQueue = + new AutoCreatedLeafQueue( + this, leafQueueName, autoCreateEnabledParentQueue); addQueue(autoCreatedLeafQueue); return autoCreatedLeafQueue; } else { - return autoQueueHandler.autoCreateQueue(placementContext); + try { + writeLock.lock(); + return autoQueueHandler.autoCreateQueue(placementContext); + } finally { + writeLock.unlock(); + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java index 1730021b22..55ab1ff269 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java @@ -31,14 +31,11 @@ */ public class CapacitySchedulerAutoQueueHandler { private final CapacitySchedulerQueueManager queueManager; - private final CapacitySchedulerConfiguration conf; private static final int MAXIMUM_DEPTH_ALLOWED = 2; public CapacitySchedulerAutoQueueHandler( - CapacitySchedulerQueueManager queueManager, - CapacitySchedulerConfiguration conf) { + CapacitySchedulerQueueManager queueManager) { this.queueManager = queueManager; - this.conf = conf; } /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java index 25b2f4d0c4..98b6d3f988 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java @@ -83,7 +83,7 @@ protected RMNodeLabelsManager createNodeLabelManager() { mockRM.start(); cs.start(); autoQueueHandler = new CapacitySchedulerAutoQueueHandler( - cs.getCapacitySchedulerQueueManager(), csConf); + cs.getCapacitySchedulerQueueManager()); mockRM.registerNode("h1:1234", MAX_MEMORY * GB); // label = x } @@ -409,7 +409,6 @@ public void testConvertDynamicParentToStaticParent() throws Exception { @Test public void testAutoQueueCreationOnAppSubmission() throws Exception { startScheduler(); - createBasicQueueStructureAndValidate(); submitApp(cs, USER0, USER0, "root.e-auto");