diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java index e0c6647f31..f6eb2adf72 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java @@ -56,7 +56,8 @@ public JAXBContextResolver() throws Exception { StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class, FairSchedulerQueueInfoList.class, AppTimeoutsInfo.class, AppTimeoutInfo.class, ResourceInformationsInfo.class, - ActivitiesInfo.class, AppActivitiesInfo.class}; + ActivitiesInfo.class, AppActivitiesInfo.class, + QueueAclsInfo.class, QueueAclInfo.class}; // these dao classes need root unwrapping final Class[] rootUnwrappedTypes = { NewApplication.class, ApplicationSubmissionContextInfo.class, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java index ca420419c8..b8a5426c77 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java @@ -61,7 +61,7 @@ public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs) { capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(), parent.getQueueResourceQuotas(), false); - queues = getQueues(parent); + queues = getQueues(cs, parent); health = new CapacitySchedulerHealthInfo(cs); } @@ -89,7 +89,8 @@ public CapacitySchedulerQueueInfoList getQueues() { return this.queues; } - protected CapacitySchedulerQueueInfoList getQueues(CSQueue parent) { + protected CapacitySchedulerQueueInfoList getQueues( + CapacityScheduler cs, CSQueue parent) { CapacitySchedulerQueueInfoList queuesInfo = new CapacitySchedulerQueueInfoList(); // JAXB marashalling leads to situation where the "type" field injected @@ -112,10 +113,10 @@ protected CapacitySchedulerQueueInfoList getQueues(CSQueue parent) { for (CSQueue queue : childQueues) { CapacitySchedulerQueueInfo info; if (queue instanceof LeafQueue) { - info = new CapacitySchedulerLeafQueueInfo((LeafQueue) queue); + info = new CapacitySchedulerLeafQueueInfo(cs, (LeafQueue) queue); } else { - info = new CapacitySchedulerQueueInfo(queue); - info.queues = getQueues(queue); + info = new CapacitySchedulerQueueInfo(cs, queue); + info.queues = getQueues(cs, queue); } queuesInfo.addToQueueInfoList(info); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java index a53e9212ee..4920f76a3e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java @@ -22,13 +22,13 @@ import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; -import javax.xml.bind.annotation.XmlTransient; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity .AutoCreatedLeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UserInfo; @@ -45,6 +45,7 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo { protected int userLimit; protected UsersInfo users; // To add another level in the XML protected float userLimitFactor; + protected float configuredMaxAMResourceLimit; protected ResourceInfo AMResourceLimit; protected ResourceInfo usedAMResource; protected ResourceInfo userAMResourceLimit; @@ -53,15 +54,14 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo { protected String defaultNodeLabelExpression; protected int defaultPriority; protected boolean isAutoCreatedLeafQueue; - - @XmlTransient - protected String orderingPolicyInfo; + protected long maxApplicationLifetime; + protected long defaultApplicationLifetime; CapacitySchedulerLeafQueueInfo() { }; - CapacitySchedulerLeafQueueInfo(LeafQueue q) { - super(q); + CapacitySchedulerLeafQueueInfo(CapacityScheduler cs, LeafQueue q) { + super(cs, q); numActiveApplications = q.getNumActiveApplications(); numPendingApplications = q.getNumPendingApplications(); numContainers = q.getNumContainers(); @@ -70,6 +70,7 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo { userLimit = q.getUserLimit(); users = new UsersInfo(q.getUsersManager().getUsersInfo()); userLimitFactor = q.getUserLimitFactor(); + configuredMaxAMResourceLimit = q.getMaxAMResourcePerQueuePercent(); AMResourceLimit = new ResourceInfo(q.getAMResourceLimit()); usedAMResource = new ResourceInfo(q.getQueueResourceUsage().getAMUsed()); preemptionDisabled = q.getPreemptionDisabled(); @@ -91,6 +92,8 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo { if ( q instanceof AutoCreatedLeafQueue) { isAutoCreatedLeafQueue = true; } + defaultApplicationLifetime = q.getDefaultApplicationLifetime(); + maxApplicationLifetime = q.getMaximumApplicationLifetime(); } @Override @@ -136,6 +139,10 @@ public UsersInfo getUsers() { public float getUserLimitFactor() { return userLimitFactor; } + + public float getConfiguredMaxAMResourceLimit() { + return configuredMaxAMResourceLimit; + } public ResourceInfo getAMResourceLimit() { return AMResourceLimit; @@ -157,10 +164,6 @@ public boolean getIntraQueuePreemptionDisabled() { return intraQueuePreemptionDisabled; } - public String getOrderingPolicyInfo() { - return orderingPolicyInfo; - } - public String getDefaultNodeLabelExpression() { return defaultNodeLabelExpression; } @@ -172,4 +175,13 @@ public int getDefaultApplicationPriority() { public boolean isAutoCreatedLeafQueue() { return isAutoCreatedLeafQueue; } + + public long getDefaultApplicationLifetime() { + return defaultApplicationLifetime; + } + + public long getMaxApplicationLifetime() { + return maxApplicationLifetime; + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index fd256b143b..bde1bb1b71 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -19,6 +19,8 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import java.util.Set; import javax.xml.bind.annotation.XmlAccessType; @@ -27,10 +29,16 @@ import javax.xml.bind.annotation.XmlSeeAlso; import javax.xml.bind.annotation.XmlTransient; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.yarn.api.records.QueueState; +import org.apache.hadoop.yarn.security.AccessType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; @@ -65,11 +73,15 @@ public class CapacitySchedulerQueueInfo { protected ResourcesInfo resources; protected ResourceInfo minEffectiveCapacity; protected ResourceInfo maxEffectiveCapacity; + protected ResourceInfo maximumAllocation; + protected QueueAclsInfo queueAcls; + protected int queuePriority; + protected String orderingPolicyInfo; CapacitySchedulerQueueInfo() { }; - CapacitySchedulerQueueInfo(CSQueue q) { + CapacitySchedulerQueueInfo(CapacityScheduler cs, CSQueue q) { queuePath = q.getQueuePath(); capacity = q.getCapacity() * 100; @@ -114,6 +126,31 @@ public class CapacitySchedulerQueueInfo { q.getQueueResourceQuotas().getEffectiveMinResource()); maxEffectiveCapacity = new ResourceInfo( q.getQueueResourceQuotas().getEffectiveMaxResource()); + maximumAllocation = new ResourceInfo(q.getMaximumAllocation()); + + CapacitySchedulerConfiguration conf = cs.getConfiguration(); + queueAcls = new QueueAclsInfo(); + for (Map.Entry e : conf + .getAcls(queueName).entrySet()) { + QueueAclInfo queueAcl = new QueueAclInfo(e.getKey().toString(), + e.getValue().getAclString()); + queueAcls.add(queueAcl); + } + + String aclApplicationMaxPriority = "acl_" + + StringUtils.toLowerCase(AccessType.APPLICATION_MAX_PRIORITY.toString()); + String priorityAcls = conf.get(queuePath + aclApplicationMaxPriority, + conf.ALL_ACL); + + QueueAclInfo queueAcl = new QueueAclInfo( + AccessType.APPLICATION_MAX_PRIORITY.toString(), priorityAcls); + queueAcls.add(queueAcl); + + queuePriority = q.getPriority().getPriority(); + if (q instanceof ParentQueue) { + orderingPolicyInfo = ((ParentQueue) q).getQueueOrderingPolicy() + .getConfigName(); + } } protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) { @@ -220,6 +257,22 @@ public ResourceInfo getMaxEffectiveCapacity(){ return maxEffectiveCapacity; } + public ResourceInfo getMaximumAllocation() { + return maximumAllocation; + } + + public QueueAclsInfo getQueueAcls() { + return queueAcls; + } + + public int getPriority() { + return queuePriority; + } + + public String getOrderingPolicyInfo() { + return orderingPolicyInfo; + } + public boolean isLeafQueue() { return getQueues() == null; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclInfo.java new file mode 100644 index 0000000000..6e623ea350 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclInfo.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "queueAcl") +@XmlAccessorType(XmlAccessType.FIELD) +public class QueueAclInfo { + + protected String accessType; + protected String accessControlList; + + public QueueAclInfo() { + // JAXB needs this + } + + public QueueAclInfo(String accessType, String accessControlList) { + this.accessType = accessType; + this.accessControlList = accessControlList; + } + + public String getAccessType() { + return accessType; + } + + public String getAccessControlList() { + return accessControlList; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclsInfo.java new file mode 100644 index 0000000000..ea07dbf8a0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclsInfo.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.ArrayList; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "queueAcls") +@XmlAccessorType(XmlAccessType.FIELD) +public class QueueAclsInfo { + + protected ArrayList queueAcl = new ArrayList(); + + public QueueAclsInfo() { + } // JAXB needs this + + public void add(QueueAclInfo queueAclInfo) { + queueAcl.add(queueAclInfo); + } + + public ArrayList getQueueAcls() { + return queueAcl; + } + + public void addAll(ArrayList queueAclsInfo) { + queueAcl.addAll(queueAclsInfo); + } + +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index d66130f6b1..25be1eb698 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -364,10 +364,10 @@ private void verifyClusterSchedulerGeneric(String type, float usedCapacity, private void verifySubQueue(JSONObject info, String q, float parentAbsCapacity, float parentAbsMaxCapacity) throws JSONException, Exception { - int numExpectedElements = 20; + int numExpectedElements = 24; boolean isParentQueue = true; if (!info.has("queues")) { - numExpectedElements = 35; + numExpectedElements = 42; isParentQueue = false; } assertEquals("incorrect number of elements", numExpectedElements, info.length());