diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index e9d04d3455..43c501f086 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -509,6 +509,9 @@ Release 2.8.0 - UNRELEASED YARN-3635. Refactored current queue mapping implementation in CapacityScheduler to use a generic PlacementManager framework. (Wangda Tan via jianhe) + YARN-4066. Large number of queues choke fair scheduler. + (Johan Gustavsson via kasha) + BUG FIXES YARN-3197. Confusing log generated by CapacityScheduler. (Varun Saxena diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java index 0092845e6a..51a298bc79 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java @@ -87,7 +87,19 @@ public void initialize(Configuration conf) throws IOException, * could be referred to as just "parent1.queue2". */ public FSLeafQueue getLeafQueue(String name, boolean create) { - FSQueue queue = getQueue(name, create, FSQueueType.LEAF); + return getLeafQueue(name, create, true); + } + + public FSLeafQueue getLeafQueue( + String name, + boolean create, + boolean recomputeSteadyShares) { + FSQueue queue = getQueue( + name, + create, + FSQueueType.LEAF, + recomputeSteadyShares + ); if (queue instanceof FSParentQueue) { return null; } @@ -117,28 +129,46 @@ public boolean removeLeafQueue(String name) { * could be referred to as just "parent1.queue2". */ public FSParentQueue getParentQueue(String name, boolean create) { - FSQueue queue = getQueue(name, create, FSQueueType.PARENT); + return getParentQueue(name, create, true); + } + + public FSParentQueue getParentQueue( + String name, + boolean create, + boolean recomputeSteadyShares) { + FSQueue queue = getQueue( + name, + create, + FSQueueType.PARENT, + recomputeSteadyShares + ); if (queue instanceof FSLeafQueue) { return null; } return (FSParentQueue) queue; } - - private FSQueue getQueue(String name, boolean create, FSQueueType queueType) { + + private FSQueue getQueue( + String name, + boolean create, + FSQueueType queueType, + boolean recomputeSteadyShares) { + boolean recompute = recomputeSteadyShares; name = ensureRootPrefix(name); + FSQueue queue; synchronized (queues) { - FSQueue queue = queues.get(name); + queue = queues.get(name); if (queue == null && create) { // if the queue doesn't exist,create it and return queue = createQueue(name, queueType); - - // Update steady fair share for all queues - if (queue != null) { - rootQueue.recomputeSteadyShares(); - } + } else { + recompute = false; } - return queue; } + if (recompute) { + rootQueue.recomputeSteadyShares(); + } + return queue; } /** @@ -376,21 +406,25 @@ private String ensureRootPrefix(String name) { public void updateAllocationConfiguration(AllocationConfiguration queueConf) { // Create leaf queues and the parent queues in a leaf's ancestry if they do not exist - for (String name : queueConf.getConfiguredQueues().get(FSQueueType.LEAF)) { - if (removeEmptyIncompatibleQueues(name, FSQueueType.LEAF)) { - getLeafQueue(name, true); + synchronized (queues) { + for (String name : queueConf.getConfiguredQueues().get( + FSQueueType.LEAF)) { + if (removeEmptyIncompatibleQueues(name, FSQueueType.LEAF)) { + getLeafQueue(name, true, false); + } + } + // At this point all leaves and 'parents with + // at least one child' would have been created. + // Now create parents with no configured leaf. + for (String name : queueConf.getConfiguredQueues().get( + FSQueueType.PARENT)) { + if (removeEmptyIncompatibleQueues(name, FSQueueType.PARENT)) { + getParentQueue(name, true, false); + } } } + rootQueue.recomputeSteadyShares(); - // At this point all leaves and 'parents with at least one child' would have been created. - // Now create parents with no configured leaf. - for (String name : queueConf.getConfiguredQueues().get( - FSQueueType.PARENT)) { - if (removeEmptyIncompatibleQueues(name, FSQueueType.PARENT)) { - getParentQueue(name, true); - } - } - for (FSQueue queue : queues.values()) { // Update queue metrics FSQueueMetrics queueMetrics = queue.getMetrics();