big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10983. Follow-up changes for YARN-10904. Contributed by Benjamin Teke

Browse Source
This commit is contained in:
9uapaw 2022-03-02 11:16:24 +01:00
parent 9199787215
commit 9e475aede6
3 changed files with 19 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
View File

@ -331,7 +331,7 @@ protected void setupQueueConfigs(Resource clusterResource) throws
// Collect and set the Node label configuration
this.queueNodeLabelsSettings = new QueueNodeLabelsSettings(configuration, parent,
getQueuePath(), queueContext.getQueueManager().getConfiguredNodeLabelsForAllQueues());
queuePath, queueContext.getQueueManager().getConfiguredNodeLabelsForAllQueues());
// Initialize the queue capacities
setupConfigurableCapacities();
@ -379,7 +379,7 @@ protected void setupQueueConfigs(Resource clusterResource) throws
// Setup application related limits
this.queueAppLifetimeSettings = new QueueAppLifetimeAndLimitSettings(configuration,
this, getQueuePath());
this, queuePath);
} finally {
writeLock.unlock();
}

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAppLifetimeAndLimitSettings.java
View File

@ -35,9 +35,9 @@ public class QueueAppLifetimeAndLimitSettings {
private int maxParallelApps;
public QueueAppLifetimeAndLimitSettings(CapacitySchedulerConfiguration configuration,
AbstractCSQueue q, String queuePath) {
AbstractCSQueue q, QueuePath queuePath) {
// Store max parallel apps property
this.maxParallelApps = configuration.getMaxParallelAppsForQueue(queuePath);
this.maxParallelApps = configuration.getMaxParallelAppsForQueue(queuePath.getFullPath());
this.maxApplicationLifetime = getInheritedMaxAppLifetime(q, configuration);
this.defaultApplicationLifetime = setupInheritedDefaultAppLifetime(q, queuePath, configuration,
maxApplicationLifetime);
@ -48,7 +48,7 @@ private long getInheritedMaxAppLifetime(CSQueue q, CapacitySchedulerConfiguratio
long maxAppLifetime = conf.getMaximumLifetimePerQueue(q.getQueuePath());
// If q is the root queue, then get max app lifetime from conf.
if (parentQ == null) {
if (q.getQueuePathObject().isRoot()) {
return maxAppLifetime;
}
@ -62,16 +62,16 @@ private long getInheritedMaxAppLifetime(CSQueue q, CapacitySchedulerConfiguratio
}
private long setupInheritedDefaultAppLifetime(CSQueue q,
String queuePath, CapacitySchedulerConfiguration conf, long myMaxAppLifetime) {
QueuePath queuePath, CapacitySchedulerConfiguration conf, long myMaxAppLifetime) {
CSQueue parentQ = q.getParent();
long defaultAppLifetime = conf.getDefaultLifetimePerQueue(queuePath);
long defaultAppLifetime = conf.getDefaultLifetimePerQueue(queuePath.getFullPath());
defaultAppLifetimeWasSpecifiedInConfig =
(defaultAppLifetime >= 0
|| (parentQ != null &&
|| (!queuePath.isRoot() &&
parentQ.getDefaultAppLifetimeWasSpecifiedInConfig()));
// If q is the root queue, then get default app lifetime from conf.
if (parentQ == null) {
if (queuePath.isRoot()) {
return defaultAppLifetime;
}

23
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueNodeLabelsSettings.java
View File

@ -19,7 +19,6 @@
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.util.Sets;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT;
import java.io.IOException;
import java.util.Set;
@ -30,14 +29,14 @@
*/
public class QueueNodeLabelsSettings {
private final CSQueue parent;
private final String queuePath;
private final QueuePath queuePath;
private Set<String> accessibleLabels;
private Set<String> configuredNodeLabels;
private String defaultLabelExpression;
public QueueNodeLabelsSettings(CapacitySchedulerConfiguration configuration,
CSQueue parent,
String queuePath,
QueuePath queuePath,
ConfiguredNodeLabels configuredNodeLabels) throws IOException {
this.parent = parent;
this.queuePath = queuePath;
@ -54,7 +53,7 @@ private void initializeNodeLabels(CapacitySchedulerConfiguration configuration,
}
private void initializeAccessibleLabels(CapacitySchedulerConfiguration configuration) {
this.accessibleLabels = configuration.getAccessibleNodeLabels(queuePath);
this.accessibleLabels = configuration.getAccessibleNodeLabels(queuePath.getFullPath());
// Inherit labels from parent if not set
if (this.accessibleLabels == null && parent != null) {
this.accessibleLabels = parent.getAccessibleNodeLabels();
@ -62,7 +61,8 @@ private void initializeAccessibleLabels(CapacitySchedulerConfiguration configura
}
private void initializeDefaultLabelExpression(CapacitySchedulerConfiguration configuration) {
this.defaultLabelExpression = configuration.getDefaultNodeLabelExpression(queuePath);
this.defaultLabelExpression = configuration.getDefaultNodeLabelExpression(
queuePath.getFullPath());
// If the accessible labels is not null and the queue has a parent with a
// similar set of labels copy the defaultNodeLabelExpression from the parent
if (this.accessibleLabels != null && parent != null
@ -75,21 +75,22 @@ private void initializeDefaultLabelExpression(CapacitySchedulerConfiguration con
private void initializeConfiguredNodeLabels(CapacitySchedulerConfiguration configuration,
ConfiguredNodeLabels configuredNodeLabelsParam) {
if (configuredNodeLabelsParam != null) {
if (queuePath.equals(ROOT)) {
if (queuePath.isRoot()) {
this.configuredNodeLabels = configuredNodeLabelsParam.getAllConfiguredLabels();
} else {
this.configuredNodeLabels = configuredNodeLabelsParam.getLabelsByQueue(queuePath);
this.configuredNodeLabels = configuredNodeLabelsParam.getLabelsByQueue(
queuePath.getFullPath());
}
} else {
// Fallback to suboptimal but correct logic
this.configuredNodeLabels = configuration.getConfiguredNodeLabels(queuePath);
this.configuredNodeLabels = configuration.getConfiguredNodeLabels(queuePath.getFullPath());
}
}
private void validateNodeLabels() throws IOException {
// Check if labels of this queue is a subset of parent queue, only do this
// when the queue in question is not root
if (isNotRoot()) {
if (!queuePath.isRoot()) {
if (parent.getAccessibleNodeLabels() != null && !parent
.getAccessibleNodeLabels().contains(RMNodeLabelsManager.ANY)) {
// If parent isn't "*", child shouldn't be "*" too
@ -109,10 +110,6 @@ private void validateNodeLabels() throws IOException {
}
}
private boolean isNotRoot() {
return parent != null && parent.getParent() != null;
}
public boolean isAccessibleToPartition(String nodePartition) {
// if queue's label is *, it can access any node
if (accessibleLabels != null && accessibleLabels.contains(RMNodeLabelsManager.ANY)) {