From d462308e0476053d1aae65eca2752eacce797e22 Mon Sep 17 00:00:00 2001 From: Sunil G Date: Sun, 3 Nov 2019 08:47:27 +0530 Subject: [PATCH] YARN-9949. Add missing queue configs for root queue in RMWebService#CapacitySchedulerInfo. Contributed by Prabhu Joseph. --- .../CapacitySchedulerConfiguration.java | 8 +++ .../AbstractComparatorOrderingPolicy.java | 3 ++ .../scheduler/policy/FairOrderingPolicy.java | 6 +++ .../scheduler/policy/FifoOrderingPolicy.java | 6 +++ .../FifoOrderingPolicyForPendingApps.java | 6 +++ ...OrderingPolicyWithExclusivePartitions.java | 7 +++ .../scheduler/policy/OrderingPolicy.java | 6 +++ .../webapp/CapacitySchedulerPage.java | 2 +- .../webapp/dao/CapacitySchedulerInfo.java | 51 +++++++++++++++++++ .../dao/CapacitySchedulerLeafQueueInfo.java | 11 +++- .../TestRMWebServicesCapacitySched.java | 42 ++++++++++++++- .../TestRMWebServicesForCSWithPartitions.java | 2 +- 12 files changed, 146 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index a72e8070f5..adf5e6b81e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -51,6 +51,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.MultiNodePolicySpec; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FairOrderingPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FifoOrderingPolicy; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FifoOrderingPolicyForPendingApps; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FifoOrderingPolicyWithExclusivePartitions; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.OrderingPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.SchedulableEntity; @@ -166,6 +167,9 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur public static final String FIFO_WITH_PARTITIONS_APP_ORDERING_POLICY = "fifo-with-partitions"; + public static final String FIFO_FOR_PENDING_APPS + = "fifo-for-pending-apps"; + public static final String DEFAULT_APP_ORDERING_POLICY = FIFO_APP_ORDERING_POLICY; @@ -579,6 +583,10 @@ public OrderingPolicy getAppOrderingPolicy( if (policyType.trim().equals(FIFO_WITH_PARTITIONS_APP_ORDERING_POLICY)) { policyType = FifoOrderingPolicyWithExclusivePartitions.class.getName(); } + if (policyType.trim().equals(FIFO_FOR_PENDING_APPS)) { + policyType = FifoOrderingPolicyForPendingApps.class.getName(); + } + try { orderingPolicy = (OrderingPolicy) Class.forName(policyType).newInstance(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/AbstractComparatorOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/AbstractComparatorOrderingPolicy.java index 4ee4450651..b5e870bf34 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/AbstractComparatorOrderingPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/AbstractComparatorOrderingPolicy.java @@ -139,4 +139,7 @@ public abstract void containerReleased(S schedulableEntity, @Override public abstract String getInfo(); + @Override + public abstract String getConfigName(); + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FairOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FairOrderingPolicy.java index 3371df803f..1f4522ce3f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FairOrderingPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FairOrderingPolicy.java @@ -23,6 +23,7 @@ import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; @@ -119,4 +120,9 @@ public String getInfo() { return "FairOrderingPolicy" + sbw; } + @Override + public String getConfigName() { + return CapacitySchedulerConfiguration.FAIR_APP_ORDERING_POLICY; + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoOrderingPolicy.java index 2d066bb432..6e545a029b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoOrderingPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoOrderingPolicy.java @@ -21,6 +21,7 @@ import java.util.*; import java.util.concurrent.ConcurrentSkipListSet; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; /** @@ -61,5 +62,10 @@ public void demandUpdated(S schedulableEntity) { public String getInfo() { return "FifoOrderingPolicy"; } + + @Override + public String getConfigName() { + return CapacitySchedulerConfiguration.FIFO_APP_ORDERING_POLICY; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoOrderingPolicyForPendingApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoOrderingPolicyForPendingApps.java index 6ced9e2dcb..0007033e9b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoOrderingPolicyForPendingApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoOrderingPolicyForPendingApps.java @@ -20,6 +20,7 @@ import java.util.*; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import java.util.concurrent.ConcurrentSkipListSet; @@ -55,6 +56,11 @@ public String getInfo() { return "FifoOrderingPolicyForPendingApps"; } + @Override + public String getConfigName() { + return CapacitySchedulerConfiguration.FIFO_FOR_PENDING_APPS; + } + @Override public void configure(Map conf) { } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoOrderingPolicyWithExclusivePartitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoOrderingPolicyWithExclusivePartitions.java index 1b1ef6606a..e6f6139305 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoOrderingPolicyWithExclusivePartitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/FifoOrderingPolicyWithExclusivePartitions.java @@ -23,6 +23,7 @@ import java.util.Iterator; import java.util.Map; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; @@ -136,6 +137,12 @@ public String getInfo() { return "FifoOrderingPolicyWithExclusivePartitions"; } + @Override + public String getConfigName() { + return CapacitySchedulerConfiguration + .FIFO_WITH_PARTITIONS_APP_ORDERING_POLICY; + } + private OrderingPolicy getPartitionOrderingPolicy(String partition) { String keyPartition = orderingPolicies.containsKey(partition) ? partition : DEFAULT_PARTITION; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/OrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/OrderingPolicy.java index 66b6a59ec4..5c622282b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/OrderingPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/OrderingPolicy.java @@ -129,4 +129,10 @@ public interface OrderingPolicy { */ public String getInfo(); + /** + * Return configuration name (which will be used to set ordering policy). + * @return configuration name + */ + String getConfigName(); + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java index 8f68e83afd..4a58545f08 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java @@ -201,7 +201,7 @@ private void renderCommonLeafQueueInfo(ResponseInfo ri) { __("Configured Minimum User Limit Percent:", Integer.toString(lqinfo.getUserLimit()) + "%"). __("Configured User Limit Factor:", lqinfo.getUserLimitFactor()). __("Accessible Node Labels:", StringUtils.join(",", lqinfo.getNodeLabels())). - __("Ordering Policy: ", lqinfo.getOrderingPolicyInfo()). + __("Ordering Policy: ", lqinfo.getOrderingPolicyDisplayName()). __("Preemption:", lqinfo.getPreemptionDisabled() ? "disabled" : "enabled"). __("Intra-queue Preemption:", lqinfo.getIntraQueuePreemptionDisabled() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java index b8a5426c77..8de752901f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java @@ -24,11 +24,17 @@ import javax.xml.bind.annotation.XmlTransient; import javax.xml.bind.annotation.XmlType; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.security.authorize.AccessControlList; +import org.apache.hadoop.yarn.security.AccessType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue; import java.util.ArrayList; +import java.util.Map; import java.util.List; @XmlRootElement(name = "capacityScheduler") @@ -43,6 +49,10 @@ public class CapacitySchedulerInfo extends SchedulerInfo { protected CapacitySchedulerQueueInfoList queues; protected QueueCapacitiesInfo capacities; protected CapacitySchedulerHealthInfo health; + protected ResourceInfo maximumAllocation; + protected QueueAclsInfo queueAcls; + protected int queuePriority; + protected String orderingPolicyInfo; @XmlTransient static final float EPSILON = 1e-8f; @@ -63,6 +73,31 @@ public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs) { parent.getQueueResourceQuotas(), false); queues = getQueues(cs, parent); health = new CapacitySchedulerHealthInfo(cs); + maximumAllocation = new ResourceInfo(parent.getMaximumAllocation()); + + CapacitySchedulerConfiguration conf = cs.getConfiguration(); + queueAcls = new QueueAclsInfo(); + for (Map.Entry e : conf + .getAcls(queueName).entrySet()) { + QueueAclInfo queueAcl = new QueueAclInfo(e.getKey().toString(), + e.getValue().getAclString()); + queueAcls.add(queueAcl); + } + + String aclApplicationMaxPriority = "acl_" + + StringUtils.toLowerCase(AccessType.APPLICATION_MAX_PRIORITY.toString()); + String priorityAcls = conf.get(parent.getQueuePath() + + aclApplicationMaxPriority, conf.ALL_ACL); + + QueueAclInfo queueAcl = new QueueAclInfo( + AccessType.APPLICATION_MAX_PRIORITY.toString(), priorityAcls); + queueAcls.add(queueAcl); + + queuePriority = parent.getPriority().getPriority(); + if (parent instanceof ParentQueue) { + orderingPolicyInfo = ((ParentQueue) parent).getQueueOrderingPolicy() + .getConfigName(); + } } public float getCapacity() { @@ -85,6 +120,22 @@ public String getQueueName() { return this.queueName; } + public ResourceInfo getMaximumAllocation() { + return maximumAllocation; + } + + public QueueAclsInfo getQueueAcls() { + return queueAcls; + } + + public int getPriority() { + return queuePriority; + } + + public String getOrderingPolicyInfo() { + return orderingPolicyInfo; + } + public CapacitySchedulerQueueInfoList getQueues() { return this.queues; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java index 4920f76a3e..6aef01bb3d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java @@ -22,6 +22,7 @@ import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlTransient; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; @@ -57,6 +58,9 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo { protected long maxApplicationLifetime; protected long defaultApplicationLifetime; + @XmlTransient + protected String orderingPolicyDisplayName; + CapacitySchedulerLeafQueueInfo() { }; @@ -75,7 +79,8 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo { usedAMResource = new ResourceInfo(q.getQueueResourceUsage().getAMUsed()); preemptionDisabled = q.getPreemptionDisabled(); intraQueuePreemptionDisabled = q.getIntraQueuePreemptionDisabled(); - orderingPolicyInfo = q.getOrderingPolicy().getInfo(); + orderingPolicyDisplayName = q.getOrderingPolicy().getInfo(); + orderingPolicyInfo = q.getOrderingPolicy().getConfigName(); defaultNodeLabelExpression = q.getDefaultNodeLabelExpression(); defaultPriority = q.getDefaultApplicationPriority().getPriority(); ArrayList usersList = users.getUsersList(); @@ -163,6 +168,10 @@ public boolean getPreemptionDisabled() { public boolean getIntraQueuePreemptionDisabled() { return intraQueuePreemptionDisabled; } + + public String getOrderingPolicyDisplayName() { + return orderingPolicyDisplayName; + } public String getDefaultNodeLabelExpression() { return defaultNodeLabelExpression; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index 25be1eb698..0fe4cb5f3b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -87,6 +87,8 @@ private class LeafQueueInfo extends QueueInfo { int maxApplicationsPerUser; int userLimit; float userLimitFactor; + long defaultApplicationLifetime; + long maxApplicationLifetime; } private static class WebServletModule extends ServletModule { @@ -133,6 +135,8 @@ private static void setupQueueConfiguration( config.setQueues(A, new String[] {"a1", "a2"}); config.setCapacity(A1, 30); config.setMaximumCapacity(A1, 50); + config.setMaximumLifetimePerQueue(A2, 100); + config.setDefaultLifetimePerQueue(A2, 50); config.setUserLimitFactor(A1, 100.0f); config.setCapacity(A2, 70); @@ -313,6 +317,10 @@ public void verifySubQueueXML(Element qElem, String q, lqi.userLimit = WebServicesTestUtils.getXmlInt(qElem, "userLimit"); lqi.userLimitFactor = WebServicesTestUtils.getXmlFloat(qElem, "userLimitFactor"); + lqi.defaultApplicationLifetime = + WebServicesTestUtils.getXmlLong(qElem, "defaultApplicationLifetime"); + lqi.maxApplicationLifetime = + WebServicesTestUtils.getXmlLong(qElem, "maxApplicationLifetime"); verifyLeafQueueGeneric(q, lqi); } } @@ -323,7 +331,7 @@ private void verifyClusterScheduler(JSONObject json) throws JSONException, JSONObject info = json.getJSONObject("scheduler"); assertEquals("incorrect number of elements in: " + info, 1, info.length()); info = info.getJSONObject("schedulerInfo"); - assertEquals("incorrect number of elements in: " + info, 8, info.length()); + assertEquals("incorrect number of elements in: " + info, 12, info.length()); verifyClusterSchedulerGeneric(info.getString("type"), (float) info.getDouble("usedCapacity"), (float) info.getDouble("capacity"), @@ -339,6 +347,16 @@ private void verifyClusterScheduler(JSONObject json) throws JSONException, assertEquals("incorrect number of elements in: " + health, 3, lastRunDetails.length()); + JSONObject maximumAllocation = info.getJSONObject("maximumAllocation"); + assertEquals("8192", maximumAllocation.getString("memory")); + assertEquals("4", maximumAllocation.getString("vCores")); + + JSONObject queueAcls = info.getJSONObject("queueAcls"); + assertEquals(1, queueAcls.length()); + + assertEquals("0", info.getString("queuePriority")); + assertEquals("utilization", info.getString("orderingPolicyInfo")); + JSONArray arr = info.getJSONObject("queues").getJSONArray("queue"); assertEquals("incorrect number of elements in: " + arr, 2, arr.length()); @@ -393,6 +411,16 @@ private void verifySubQueue(JSONObject info, String q, String q2 = q + "." + obj.getString("queueName"); verifySubQueue(obj, q2, qi.absoluteCapacity, qi.absoluteMaxCapacity); } + + JSONObject maximumAllocation = info.getJSONObject("maximumAllocation"); + assertEquals("8192", maximumAllocation.getString("memory")); + assertEquals("4", maximumAllocation.getString("vCores")); + + JSONObject queueAcls = info.getJSONObject("queueAcls"); + assertEquals(1, queueAcls.length()); + + assertEquals("0", info.getString("queuePriority")); + assertEquals("utilization", info.getString("orderingPolicyInfo")); } else { Assert.assertEquals("\"type\" field is incorrect", "capacitySchedulerLeafQueueInfo", info.getString("type")); @@ -404,6 +432,9 @@ private void verifySubQueue(JSONObject info, String q, lqi.maxApplicationsPerUser = info.getInt("maxApplicationsPerUser"); lqi.userLimit = info.getInt("userLimit"); lqi.userLimitFactor = (float) info.getDouble("userLimitFactor"); + lqi.defaultApplicationLifetime = + info.getLong("defaultApplicationLifetime"); + lqi.maxApplicationLifetime = info.getLong("maxApplicationLifetime"); verifyLeafQueueGeneric(q, lqi); // resourcesUsed and users (per-user resources used) are checked in // testPerUserResource() @@ -468,6 +499,15 @@ private void verifyLeafQueueGeneric(String q, LeafQueueInfo info) info.userLimit); assertEquals("userLimitFactor doesn't match", csConf.getUserLimitFactor(q), info.userLimitFactor, 1e-3f); + + if (q.equals("root.a.a2")) { + assertEquals("defaultApplicationLifetime doesn't match", + csConf.getDefaultLifetimePerQueue(q), + info.defaultApplicationLifetime); + assertEquals("maxApplicationLifetime doesn't match", + csConf.getMaximumLifetimePerQueue(q), + info.maxApplicationLifetime); + } } //Return a child Node of node with the tagname or null if none exists diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java index 46b809c314..7c3ec6cdb3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java @@ -563,7 +563,7 @@ private void verifySchedulerInfoJson(JSONObject json) JSONObject info = json.getJSONObject("scheduler"); assertEquals("incorrect number of elements", 1, info.length()); info = info.getJSONObject("schedulerInfo"); - assertEquals("incorrect number of elements", 8, info.length()); + assertEquals("incorrect number of elements", 12, info.length()); JSONObject capacitiesJsonObject = info.getJSONObject(CAPACITIES); JSONArray partitionsCapsArray = capacitiesJsonObject.getJSONArray(QUEUE_CAPACITIES_BY_PARTITION);