big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10237. Add isAbsoluteResource config for queue in scheduler response. Contributed by Prabhu Joseph

Browse Source
This commit is contained in:
Szilard Nemeth 2020-04-24 17:04:47 +02:00
parent 30ef8d0f1a
commit d4874585f4
3 changed files with 34 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
View File

@ -377,7 +377,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
public static final String PATTERN_FOR_ABSOLUTE_RESOURCE = "^\\[[\\w\\.,\\-_=\\ /]+\\]$";
private static final Pattern RESOURCE_PATTERN = Pattern.compile(PATTERN_FOR_ABSOLUTE_RESOURCE);
public static final Pattern RESOURCE_PATTERN = Pattern.compile(PATTERN_FOR_ABSOLUTE_RESOURCE);
/**
* Different resource types supported.

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
View File

@ -22,6 +22,7 @@
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@ -42,6 +43,11 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.
CapacitySchedulerConfiguration.RESOURCE_PATTERN;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.
CapacitySchedulerConfiguration.CAPACITY;
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
@XmlSeeAlso({CapacitySchedulerLeafQueueInfo.class})
@ -61,6 +67,7 @@ public class CapacitySchedulerQueueInfo {
protected float absoluteUsedCapacity;
protected int numApplications;
protected String queueName;
protected boolean isAbsoluteResource;
protected QueueState state;
protected CapacitySchedulerQueueInfoList queues;
protected ResourceInfo resourcesUsed;
@ -151,6 +158,11 @@ public class CapacitySchedulerQueueInfo {
orderingPolicyInfo = ((ParentQueue) q).getQueueOrderingPolicy()
.getConfigName();
}
String configuredCapacity = conf.get(
CapacitySchedulerConfiguration.getQueuePrefix(queuePath) + CAPACITY);
isAbsoluteResource = (configuredCapacity != null)
&& RESOURCE_PATTERN.matcher(configuredCapacity).find();
}
protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) {
@ -203,6 +215,10 @@ public long getPendingContainers() {
return pendingContainers;
}
public boolean isAbsoluteResource() {
return isAbsoluteResource;
}
public String getQueueName() {
return this.queueName;
}

19
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
View File

@ -20,6 +20,7 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@ -79,6 +80,7 @@ private class QueueInfo {
int numApplications;
String queueName;
String state;
boolean isAbsoluteResource;
}
private class LeafQueueInfo extends QueueInfo {
@ -131,6 +133,9 @@ private static void setupQueueConfiguration(
final String B = CapacitySchedulerConfiguration.ROOT + ".b";
config.setCapacity(B, 89.5f);
final String C = CapacitySchedulerConfiguration.ROOT + ".c";
config.setCapacity(C, "[memory=1024]");
// Define 2nd-level queues
final String A1 = A + ".a1";
final String A2 = A + ".a2";
@ -290,6 +295,8 @@ public void verifySubQueueXML(Element qElem, String q,
WebServicesTestUtils.getXmlInt(qElem, "numApplications");
qi.queueName = WebServicesTestUtils.getXmlString(qElem, "queueName");
qi.state = WebServicesTestUtils.getXmlString(qElem, "state");
qi.isAbsoluteResource = WebServicesTestUtils.getXmlBoolean(qElem,
"isAbsoluteResource");
verifySubQueueGeneric(q, qi, parentAbsCapacity, parentAbsMaxCapacity);
if (hasSubQueues) {
for (int j = 0; j < children.getLength(); j++) {
@ -384,10 +391,10 @@ private void verifyClusterSchedulerGeneric(String type, float usedCapacity,
private void verifySubQueue(JSONObject info, String q,
float parentAbsCapacity, float parentAbsMaxCapacity)
throws JSONException, Exception {
int numExpectedElements = 24;
int numExpectedElements = 25;
boolean isParentQueue = true;
if (!info.has("queues")) {
numExpectedElements = 42;
numExpectedElements = 43;
isParentQueue = false;
}
assertEquals("incorrect number of elements", numExpectedElements, info.length());
@ -471,6 +478,14 @@ private void verifySubQueueGeneric(String q, QueueInfo info,
+ " expected: " + q, qshortName.matches(info.queueName));
assertTrue("state doesn't match",
(csConf.getState(q).toString()).matches(info.state));
if (q.equals("c")) {
assertTrue("c queue is not configured in Absolute resource",
info.isAbsoluteResource);
} else {
assertFalse(info.queueName
+ " queue is not configured in Absolute resource",
info.isAbsoluteResource);
}
}