big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-11033. isAbsoluteResource is not correct for dynamically created queues. Contributed by Tamas Domok

Browse Source
This commit is contained in:
Szilard Nemeth 2022-03-02 14:45:31 +01:00
parent f06f44b1c2
commit 3ef3c5a05b
5 changed files with 13 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
View File

@ -128,7 +128,7 @@ public abstract class AbstractCSQueue implements CSQueue {
// either at this level or anywhere in the queue's hierarchy.
private volatile boolean defaultAppLifetimeWasSpecifiedInConfig = false;
protected enum CapacityConfigType {
public enum CapacityConfigType {
NONE, PERCENTAGE, ABSOLUTE_RESOURCE
};
protected CapacityConfigType capacityConfigType =

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
View File

@ -27,6 +27,7 @@
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.security.AccessType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
@ -46,6 +47,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
protected float usedCapacity;
protected float maxCapacity;
protected String queueName;
private boolean isAbsoluteResource;
protected CapacitySchedulerQueueInfoList queues;
protected QueueCapacitiesInfo capacities;
protected CapacitySchedulerHealthInfo health;
@ -75,6 +77,9 @@ public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs) {
health = new CapacitySchedulerHealthInfo(cs);
maximumAllocation = new ResourceInfo(parent.getMaximumAllocation());
isAbsoluteResource = parent.getCapacityConfigType() ==
AbstractCSQueue.CapacityConfigType.ABSOLUTE_RESOURCE;
CapacitySchedulerConfiguration conf = cs.getConfiguration();
queueAcls = new QueueAclsInfo();
for (Map.Entry<AccessType, AccessControlList> e : conf

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
View File

@ -35,6 +35,7 @@
import org.apache.hadoop.yarn.security.AccessType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
@ -42,11 +43,6 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.
CapacitySchedulerConfiguration.RESOURCE_PATTERN;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.
CapacitySchedulerConfiguration.CAPACITY;
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
@XmlSeeAlso({CapacitySchedulerLeafQueueInfo.class})
@ -160,10 +156,8 @@ public class CapacitySchedulerQueueInfo {
.getConfigName();
}
String configuredCapacity = conf.get(
CapacitySchedulerConfiguration.getQueuePrefix(queuePath) + CAPACITY);
isAbsoluteResource = (configuredCapacity != null)
&& RESOURCE_PATTERN.matcher(configuredCapacity).find();
isAbsoluteResource = q.getCapacityConfigType() ==
AbstractCSQueue.CapacityConfigType.ABSOLUTE_RESOURCE;
autoCreateChildQueueEnabled = conf.
isAutoCreateChildQueueEnabled(queuePath);

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
View File

@ -362,11 +362,13 @@ private void verifyClusterScheduler(JSONObject json) throws JSONException,
JSONObject info = json.getJSONObject("scheduler");
assertEquals("incorrect number of elements in: " + info, 1, info.length());
info = info.getJSONObject("schedulerInfo");
assertEquals("incorrect number of elements in: " + info, 12, info.length());
assertEquals("incorrect number of elements in: " + info, 13, info.length());
verifyClusterSchedulerGeneric(info.getString("type"),
(float) info.getDouble("usedCapacity"),
(float) info.getDouble("capacity"),
(float) info.getDouble("maxCapacity"), info.getString("queueName"));
assertFalse("isAbsoluteResource should be null for root queue",
info.getBoolean("isAbsoluteResource"));
JSONObject health = info.getJSONObject("health");
assertNotNull(health);
assertEquals("incorrect number of elements in: " + health, 3,

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java
View File

@ -576,7 +576,7 @@ private void verifySchedulerInfoJson(JSONObject json)
JSONObject info = json.getJSONObject("scheduler");
assertEquals("incorrect number of elements", 1, info.length());
info = info.getJSONObject("schedulerInfo");
assertEquals("incorrect number of elements", 12, info.length());
assertEquals("incorrect number of elements", 13, info.length());
JSONObject capacitiesJsonObject = info.getJSONObject(CAPACITIES);
JSONArray partitionsCapsArray =
capacitiesJsonObject.getJSONArray(QUEUE_CAPACITIES_BY_PARTITION);