From 9e475aede6adc5b2e31f7447e3ab94c6db27f93b Mon Sep 17 00:00:00 2001 From: 9uapaw Date: Wed, 2 Mar 2022 11:16:24 +0100 Subject: [PATCH] YARN-10983. Follow-up changes for YARN-10904. Contributed by Benjamin Teke --- .../scheduler/capacity/AbstractCSQueue.java | 4 ++-- .../QueueAppLifetimeAndLimitSettings.java | 14 +++++------ .../capacity/QueueNodeLabelsSettings.java | 23 ++++++++----------- 3 files changed, 19 insertions(+), 22 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 020c601104..e924932a36 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -331,7 +331,7 @@ protected void setupQueueConfigs(Resource clusterResource) throws // Collect and set the Node label configuration this.queueNodeLabelsSettings = new QueueNodeLabelsSettings(configuration, parent, - getQueuePath(), queueContext.getQueueManager().getConfiguredNodeLabelsForAllQueues()); + queuePath, queueContext.getQueueManager().getConfiguredNodeLabelsForAllQueues()); // Initialize the queue capacities setupConfigurableCapacities(); @@ -379,7 +379,7 @@ protected void setupQueueConfigs(Resource clusterResource) throws // Setup application related limits this.queueAppLifetimeSettings = new QueueAppLifetimeAndLimitSettings(configuration, - this, getQueuePath()); + this, queuePath); } finally { writeLock.unlock(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAppLifetimeAndLimitSettings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAppLifetimeAndLimitSettings.java index e0f4d60fe4..2e71d8eef3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAppLifetimeAndLimitSettings.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAppLifetimeAndLimitSettings.java @@ -35,9 +35,9 @@ public class QueueAppLifetimeAndLimitSettings { private int maxParallelApps; public QueueAppLifetimeAndLimitSettings(CapacitySchedulerConfiguration configuration, - AbstractCSQueue q, String queuePath) { + AbstractCSQueue q, QueuePath queuePath) { // Store max parallel apps property - this.maxParallelApps = configuration.getMaxParallelAppsForQueue(queuePath); + this.maxParallelApps = configuration.getMaxParallelAppsForQueue(queuePath.getFullPath()); this.maxApplicationLifetime = getInheritedMaxAppLifetime(q, configuration); this.defaultApplicationLifetime = setupInheritedDefaultAppLifetime(q, queuePath, configuration, maxApplicationLifetime); @@ -48,7 +48,7 @@ private long getInheritedMaxAppLifetime(CSQueue q, CapacitySchedulerConfiguratio long maxAppLifetime = conf.getMaximumLifetimePerQueue(q.getQueuePath()); // If q is the root queue, then get max app lifetime from conf. - if (parentQ == null) { + if (q.getQueuePathObject().isRoot()) { return maxAppLifetime; } @@ -62,16 +62,16 @@ private long getInheritedMaxAppLifetime(CSQueue q, CapacitySchedulerConfiguratio } private long setupInheritedDefaultAppLifetime(CSQueue q, - String queuePath, CapacitySchedulerConfiguration conf, long myMaxAppLifetime) { + QueuePath queuePath, CapacitySchedulerConfiguration conf, long myMaxAppLifetime) { CSQueue parentQ = q.getParent(); - long defaultAppLifetime = conf.getDefaultLifetimePerQueue(queuePath); + long defaultAppLifetime = conf.getDefaultLifetimePerQueue(queuePath.getFullPath()); defaultAppLifetimeWasSpecifiedInConfig = (defaultAppLifetime >= 0 - || (parentQ != null && + || (!queuePath.isRoot() && parentQ.getDefaultAppLifetimeWasSpecifiedInConfig())); // If q is the root queue, then get default app lifetime from conf. - if (parentQ == null) { + if (queuePath.isRoot()) { return defaultAppLifetime; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueNodeLabelsSettings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueNodeLabelsSettings.java index 8d64e17a5e..c431d2bb45 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueNodeLabelsSettings.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueNodeLabelsSettings.java @@ -19,7 +19,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.util.Sets; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; -import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT; import java.io.IOException; import java.util.Set; @@ -30,14 +29,14 @@ */ public class QueueNodeLabelsSettings { private final CSQueue parent; - private final String queuePath; + private final QueuePath queuePath; private Set accessibleLabels; private Set configuredNodeLabels; private String defaultLabelExpression; public QueueNodeLabelsSettings(CapacitySchedulerConfiguration configuration, CSQueue parent, - String queuePath, + QueuePath queuePath, ConfiguredNodeLabels configuredNodeLabels) throws IOException { this.parent = parent; this.queuePath = queuePath; @@ -54,7 +53,7 @@ private void initializeNodeLabels(CapacitySchedulerConfiguration configuration, } private void initializeAccessibleLabels(CapacitySchedulerConfiguration configuration) { - this.accessibleLabels = configuration.getAccessibleNodeLabels(queuePath); + this.accessibleLabels = configuration.getAccessibleNodeLabels(queuePath.getFullPath()); // Inherit labels from parent if not set if (this.accessibleLabels == null && parent != null) { this.accessibleLabels = parent.getAccessibleNodeLabels(); @@ -62,7 +61,8 @@ private void initializeAccessibleLabels(CapacitySchedulerConfiguration configura } private void initializeDefaultLabelExpression(CapacitySchedulerConfiguration configuration) { - this.defaultLabelExpression = configuration.getDefaultNodeLabelExpression(queuePath); + this.defaultLabelExpression = configuration.getDefaultNodeLabelExpression( + queuePath.getFullPath()); // If the accessible labels is not null and the queue has a parent with a // similar set of labels copy the defaultNodeLabelExpression from the parent if (this.accessibleLabels != null && parent != null @@ -75,21 +75,22 @@ private void initializeDefaultLabelExpression(CapacitySchedulerConfiguration con private void initializeConfiguredNodeLabels(CapacitySchedulerConfiguration configuration, ConfiguredNodeLabels configuredNodeLabelsParam) { if (configuredNodeLabelsParam != null) { - if (queuePath.equals(ROOT)) { + if (queuePath.isRoot()) { this.configuredNodeLabels = configuredNodeLabelsParam.getAllConfiguredLabels(); } else { - this.configuredNodeLabels = configuredNodeLabelsParam.getLabelsByQueue(queuePath); + this.configuredNodeLabels = configuredNodeLabelsParam.getLabelsByQueue( + queuePath.getFullPath()); } } else { // Fallback to suboptimal but correct logic - this.configuredNodeLabels = configuration.getConfiguredNodeLabels(queuePath); + this.configuredNodeLabels = configuration.getConfiguredNodeLabels(queuePath.getFullPath()); } } private void validateNodeLabels() throws IOException { // Check if labels of this queue is a subset of parent queue, only do this // when the queue in question is not root - if (isNotRoot()) { + if (!queuePath.isRoot()) { if (parent.getAccessibleNodeLabels() != null && !parent .getAccessibleNodeLabels().contains(RMNodeLabelsManager.ANY)) { // If parent isn't "*", child shouldn't be "*" too @@ -109,10 +110,6 @@ private void validateNodeLabels() throws IOException { } } - private boolean isNotRoot() { - return parent != null && parent.getParent() != null; - } - public boolean isAccessibleToPartition(String nodePartition) { // if queue's label is *, it can access any node if (accessibleLabels != null && accessibleLabels.contains(RMNodeLabelsManager.ANY)) {