YARN-10578. Fix Auto Queue Creation parent handling. Contributed by Andras Gyori

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java

@@ -154,11 +154,6 @@ public enum CapacityConfigType {
   // is it a dynamic queue?
   private boolean dynamicQueue = false;
 
-  // When this queue has application submit to?
-  // This property only applies to dynamic queue,
-  // and will be used to check when the queue need to be removed.
-  private long lastSubmittedTimestamp;
-
   public AbstractCSQueue(CapacitySchedulerContext cs,
       String queueName, CSQueue parent, CSQueue old) throws IOException {
     this(cs, cs.getConfiguration(), queueName, parent, old);
@@ -1633,18 +1628,4 @@ public void setDynamicQueue(boolean dynamicQueue) {
       writeLock.unlock();
     }
   }
-
-  public long getLastSubmittedTimestamp() {
-    return lastSubmittedTimestamp;
-  }
-
-  // "Tab" the queue, so this queue won't be removed because of idle timeout.
-  public void signalToSubmitToQueue() {
-    writeLock.lock();
-    try {
-      this.lastSubmittedTimestamp = System.currentTimeMillis();
-    } finally {
-      writeLock.unlock();
-    }
-  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java

@@ -343,7 +343,7 @@ void initScheduler(Configuration configuration) throws
       this.queueManager.setCapacitySchedulerContext(this);
 
       this.autoQueueHandler = new CapacitySchedulerAutoQueueHandler(
-          this.queueManager, this.conf);
+          this.queueManager);
 
       this.workflowPriorityMappingsMgr = new WorkflowPriorityMappingsManager();
 
@@ -3380,26 +3380,25 @@ private LeafQueue autoCreateLeafQueue(
     if (!StringUtils.isEmpty(parentQueueName)) {
       CSQueue parentQueue = getQueue(parentQueueName);
 
-      if (parentQueue == null) {
-        throw new SchedulerDynamicEditException(
-            "Could not auto-create leaf queue for " + leafQueueName
-                + ". Queue mapping specifies an invalid parent queue "
-                + "which does not exist " + parentQueueName);
-      }
-
-      if (conf.isAutoCreateChildQueueEnabled(parentQueue.getQueuePath())) {
+      if (parentQueue != null &&
+          conf.isAutoCreateChildQueueEnabled(parentQueue.getQueuePath())) {
         // Case 1: Handle ManagedParentQueue
-        AutoCreatedLeafQueue autoCreatedLeafQueue = null;
         ManagedParentQueue autoCreateEnabledParentQueue =
             (ManagedParentQueue) parentQueue;
-        autoCreatedLeafQueue = new AutoCreatedLeafQueue(this, leafQueueName,
-            autoCreateEnabledParentQueue);
+        AutoCreatedLeafQueue autoCreatedLeafQueue =
+            new AutoCreatedLeafQueue(
+                this, leafQueueName, autoCreateEnabledParentQueue);
 
         addQueue(autoCreatedLeafQueue);
         return autoCreatedLeafQueue;
 
       } else {
-        return autoQueueHandler.autoCreateQueue(placementContext);
+        try {
+          writeLock.lock();
+          return autoQueueHandler.autoCreateQueue(placementContext);
+        } finally {
+          writeLock.unlock();
+        }
       }
     }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java

@@ -31,14 +31,11 @@
  */
 public class CapacitySchedulerAutoQueueHandler {
   private final CapacitySchedulerQueueManager queueManager;
-  private final CapacitySchedulerConfiguration conf;
   private static final int MAXIMUM_DEPTH_ALLOWED = 2;
 
   public CapacitySchedulerAutoQueueHandler(
-      CapacitySchedulerQueueManager queueManager,
-      CapacitySchedulerConfiguration conf) {
+      CapacitySchedulerQueueManager queueManager) {
     this.queueManager = queueManager;
-    this.conf = conf;
   }
 
   /**

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java

@@ -83,7 +83,7 @@ protected RMNodeLabelsManager createNodeLabelManager() {
     mockRM.start();
     cs.start();
     autoQueueHandler = new CapacitySchedulerAutoQueueHandler(
-        cs.getCapacitySchedulerQueueManager(), csConf);
+        cs.getCapacitySchedulerQueueManager());
     mockRM.registerNode("h1:1234", MAX_MEMORY * GB); // label = x
   }
 
@@ -409,7 +409,6 @@ public void testConvertDynamicParentToStaticParent() throws Exception {
   @Test
   public void testAutoQueueCreationOnAppSubmission() throws Exception {
     startScheduler();
-    createBasicQueueStructureAndValidate();
 
     submitApp(cs, USER0, USER0, "root.e-auto");