YARN-10655. Limit queue creation depth relative to its first static parent. Contributed by Andras Gyori.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerAutoQueueHandler.java

@@ -55,34 +55,43 @@ public LeafQueue autoCreateQueue(ApplicationPlacementContext queue)
     List<ApplicationPlacementContext> parentsToCreate = new ArrayList<>();
 
     ApplicationPlacementContext queueCandidateContext = parentContext;
-    CSQueue existingQueueCandidate = getQueue(
+    CSQueue firstExistingQueue = getQueue(
         queueCandidateContext.getFullQueuePath());
 
-    while (existingQueueCandidate == null) {
+    while (firstExistingQueue == null) {
       parentsToCreate.add(queueCandidateContext);
       queueCandidateContext = CSQueueUtils.extractQueuePath(
           queueCandidateContext.getParentQueue());
-      existingQueueCandidate = getQueue(
+      firstExistingQueue = getQueue(
           queueCandidateContext.getFullQueuePath());
     }
 
+    CSQueue firstExistingStaticQueue = firstExistingQueue;
+    // Include the LeafQueue in the distance
+    int firstStaticParentDistance = parentsToCreate.size() + 1;
+
+    while(isNonStaticParent(firstExistingStaticQueue)) {
+      queueCandidateContext = CSQueueUtils.extractQueuePath(
+          queueCandidateContext.getParentQueue());
+      firstExistingStaticQueue = getQueue(
+          queueCandidateContext.getFullQueuePath());
+      ++firstStaticParentDistance;
+    }
+
     // Reverse the collection to to represent the hierarchy to be created
     // from highest to lowest level
     Collections.reverse(parentsToCreate);
 
-    if (!(existingQueueCandidate instanceof ParentQueue)) {
+    if (!(firstExistingQueue instanceof ParentQueue)) {
       throw new SchedulerDynamicEditException(
           "Could not auto create hierarchy of "
               + queue.getFullQueuePath() + ". Queue "
-              + existingQueueCandidate.getQueuePath() +
+              + firstExistingQueue.getQueuePath() +
               " is not a ParentQueue."
       );
     }
-    ParentQueue existingParentQueue = (ParentQueue) existingQueueCandidate;
+    ParentQueue existingParentQueue = (ParentQueue) firstExistingQueue;
     int depthLimit = extractDepthLimit(existingParentQueue);
-    // The number of levels to be created including the LeafQueue
-    // (which is last)
-    int levelsToCreate = parentsToCreate.size() + 1;
 
     if (depthLimit == 0) {
       throw new SchedulerDynamicEditException("Auto creation of queue " +
@@ -90,12 +99,12 @@ public LeafQueue autoCreateQueue(ApplicationPlacementContext queue)
           + existingParentQueue.getQueuePath());
     }
 
-    if (levelsToCreate > depthLimit) {
+    if (firstStaticParentDistance > depthLimit) {
       throw new SchedulerDynamicEditException(
           "Could not auto create queue " + queue.getFullQueuePath()
-              + ". In order to create the desired queue hierarchy, " +
-              levelsToCreate + " levels of queues would need " +
-              "to be created, which is above the limit.");
+              + ". The distance of the LeafQueue from the first static " +
+              "ParentQueue is" + firstStaticParentDistance + ", which is " +
+              "above the limit.");
     }
 
     for (ApplicationPlacementContext current : parentsToCreate) {
@@ -123,4 +132,9 @@ private int extractDepthLimit(ParentQueue parentQueue) {
   private CSQueue getQueue(String queue) {
     return queue != null ? queueManager.getQueue(queue) : null;
   }
+
+  private boolean isNonStaticParent(CSQueue queue) {
+    return (!(queue instanceof AbstractCSQueue)
+        || ((AbstractCSQueue) queue).isDynamicQueue());
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java

@@ -565,6 +565,26 @@ public void testAutoCreateQueueMaxQueuesLimit() throws Exception {
     }
   }
 
+  @Test
+  public void testAutoQueueCreationDepthLimitFromStaticParent()
+      throws Exception {
+    startScheduler();
+
+    // a is the first existing queue here and it is static, therefore
+    // the distance is 2
+    createQueue("root.a.a-auto.a1-auto");
+    Assert.assertNotNull(cs.getQueue("root.a.a-auto.a1-auto"));
+
+    try {
+      createQueue("root.a.a-auto.a2-auto.a3-auto");
+      Assert.fail("Queue creation should not succeed because the distance " +
+          "from the first static parent is above limit");
+    } catch (SchedulerDynamicEditException ignored) {
+
+    }
+
+  }
+
   private LeafQueue createQueue(String queuePath) throws YarnException {
     return autoQueueHandler.autoCreateQueue(
         CSQueueUtils.extractQueuePath(queuePath));