big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-9937. Add missing queue configs in RMWebService#CapacitySchedulerQueueInfo. Contributed by Prabhu Joseph.

Browse Source
This commit is contained in:
Sunil G 2019-10-31 00:34:27 +05:30
parent e3e7daa4f5
commit 9a2e43e29e
7 changed files with 181 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
View File

@ -56,7 +56,8 @@ public JAXBContextResolver() throws Exception {
StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class,
FairSchedulerQueueInfoList.class, AppTimeoutsInfo.class,
AppTimeoutInfo.class, ResourceInformationsInfo.class,
ActivitiesInfo.class, AppActivitiesInfo.class};
ActivitiesInfo.class, AppActivitiesInfo.class,
QueueAclsInfo.class, QueueAclInfo.class};
// these dao classes need root unwrapping
final Class[] rootUnwrappedTypes =
{ NewApplication.class, ApplicationSubmissionContextInfo.class,

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
View File

@ -61,7 +61,7 @@ public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs) {
capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(),
parent.getQueueResourceQuotas(), false);
queues = getQueues(parent);
queues = getQueues(cs, parent);
health = new CapacitySchedulerHealthInfo(cs);
}
@ -89,7 +89,8 @@ public CapacitySchedulerQueueInfoList getQueues() {
return this.queues;
}
protected CapacitySchedulerQueueInfoList getQueues(CSQueue parent) {
protected CapacitySchedulerQueueInfoList getQueues(
CapacityScheduler cs, CSQueue parent) {
CapacitySchedulerQueueInfoList queuesInfo =
new CapacitySchedulerQueueInfoList();
// JAXB marashalling leads to situation where the "type" field injected
@ -112,10 +113,10 @@ protected CapacitySchedulerQueueInfoList getQueues(CSQueue parent) {
for (CSQueue queue : childQueues) {
CapacitySchedulerQueueInfo info;
if (queue instanceof LeafQueue) {
info = new CapacitySchedulerLeafQueueInfo((LeafQueue) queue);
info = new CapacitySchedulerLeafQueueInfo(cs, (LeafQueue) queue);
} else {
info = new CapacitySchedulerQueueInfo(queue);
info.queues = getQueues(queue);
info = new CapacitySchedulerQueueInfo(cs, queue);
info.queues = getQueues(cs, queue);
}
queuesInfo.addToQueueInfoList(info);
}

32
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
View File

@ -22,13 +22,13 @@
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlTransient;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
.AutoCreatedLeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UserInfo;
@ -45,6 +45,7 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
protected int userLimit;
protected UsersInfo users; // To add another level in the XML
protected float userLimitFactor;
protected float configuredMaxAMResourceLimit;
protected ResourceInfo AMResourceLimit;
protected ResourceInfo usedAMResource;
protected ResourceInfo userAMResourceLimit;
@ -53,15 +54,14 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
protected String defaultNodeLabelExpression;
protected int defaultPriority;
protected boolean isAutoCreatedLeafQueue;
@XmlTransient
protected String orderingPolicyInfo;
protected long maxApplicationLifetime;
protected long defaultApplicationLifetime;
CapacitySchedulerLeafQueueInfo() {
};
CapacitySchedulerLeafQueueInfo(LeafQueue q) {
super(q);
CapacitySchedulerLeafQueueInfo(CapacityScheduler cs, LeafQueue q) {
super(cs, q);
numActiveApplications = q.getNumActiveApplications();
numPendingApplications = q.getNumPendingApplications();
numContainers = q.getNumContainers();
@ -70,6 +70,7 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
userLimit = q.getUserLimit();
users = new UsersInfo(q.getUsersManager().getUsersInfo());
userLimitFactor = q.getUserLimitFactor();
configuredMaxAMResourceLimit = q.getMaxAMResourcePerQueuePercent();
AMResourceLimit = new ResourceInfo(q.getAMResourceLimit());
usedAMResource = new ResourceInfo(q.getQueueResourceUsage().getAMUsed());
preemptionDisabled = q.getPreemptionDisabled();
@ -91,6 +92,8 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
if ( q instanceof AutoCreatedLeafQueue) {
isAutoCreatedLeafQueue = true;
}
defaultApplicationLifetime = q.getDefaultApplicationLifetime();
maxApplicationLifetime = q.getMaximumApplicationLifetime();
}
@Override
@ -136,6 +139,10 @@ public UsersInfo getUsers() {
public float getUserLimitFactor() {
return userLimitFactor;
}
public float getConfiguredMaxAMResourceLimit() {
return configuredMaxAMResourceLimit;
}
public ResourceInfo getAMResourceLimit() {
return AMResourceLimit;
@ -157,10 +164,6 @@ public boolean getIntraQueuePreemptionDisabled() {
return intraQueuePreemptionDisabled;
}
public String getOrderingPolicyInfo() {
return orderingPolicyInfo;
}
public String getDefaultNodeLabelExpression() {
return defaultNodeLabelExpression;
}
@ -172,4 +175,13 @@ public int getDefaultApplicationPriority() {
public boolean isAutoCreatedLeafQueue() {
return isAutoCreatedLeafQueue;
}
public long getDefaultApplicationLifetime() {
return defaultApplicationLifetime;
}
public long getMaxApplicationLifetime() {
return maxApplicationLifetime;
}
}

55
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
View File

@ -19,6 +19,8 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import javax.xml.bind.annotation.XmlAccessType;
@ -27,10 +29,16 @@
import javax.xml.bind.annotation.XmlSeeAlso;
import javax.xml.bind.annotation.XmlTransient;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.security.AccessType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
@ -65,11 +73,15 @@ public class CapacitySchedulerQueueInfo {
protected ResourcesInfo resources;
protected ResourceInfo minEffectiveCapacity;
protected ResourceInfo maxEffectiveCapacity;
protected ResourceInfo maximumAllocation;
protected QueueAclsInfo queueAcls;
protected int queuePriority;
protected String orderingPolicyInfo;
CapacitySchedulerQueueInfo() {
};
CapacitySchedulerQueueInfo(CSQueue q) {
CapacitySchedulerQueueInfo(CapacityScheduler cs, CSQueue q) {
queuePath = q.getQueuePath();
capacity = q.getCapacity() * 100;
@ -114,6 +126,31 @@ public class CapacitySchedulerQueueInfo {
q.getQueueResourceQuotas().getEffectiveMinResource());
maxEffectiveCapacity = new ResourceInfo(
q.getQueueResourceQuotas().getEffectiveMaxResource());
maximumAllocation = new ResourceInfo(q.getMaximumAllocation());
CapacitySchedulerConfiguration conf = cs.getConfiguration();
queueAcls = new QueueAclsInfo();
for (Map.Entry<AccessType, AccessControlList> e : conf
.getAcls(queueName).entrySet()) {
QueueAclInfo queueAcl = new QueueAclInfo(e.getKey().toString(),
e.getValue().getAclString());
queueAcls.add(queueAcl);
}
String aclApplicationMaxPriority = "acl_" +
StringUtils.toLowerCase(AccessType.APPLICATION_MAX_PRIORITY.toString());
String priorityAcls = conf.get(queuePath + aclApplicationMaxPriority,
conf.ALL_ACL);
QueueAclInfo queueAcl = new QueueAclInfo(
AccessType.APPLICATION_MAX_PRIORITY.toString(), priorityAcls);
queueAcls.add(queueAcl);
queuePriority = q.getPriority().getPriority();
if (q instanceof ParentQueue) {
orderingPolicyInfo = ((ParentQueue) q).getQueueOrderingPolicy()
.getConfigName();
}
}
protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) {
@ -220,6 +257,22 @@ public ResourceInfo getMaxEffectiveCapacity(){
return maxEffectiveCapacity;
}
public ResourceInfo getMaximumAllocation() {
return maximumAllocation;
}
public QueueAclsInfo getQueueAcls() {
return queueAcls;
}
public int getPriority() {
return queuePriority;
}
public String getOrderingPolicyInfo() {
return orderingPolicyInfo;
}
public boolean isLeafQueue() {
return getQueues() == null;
}

48
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclInfo.java Normal file
View File

@ -0,0 +1,48 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
@XmlRootElement(name = "queueAcl")
@XmlAccessorType(XmlAccessType.FIELD)
public class QueueAclInfo {
protected String accessType;
protected String accessControlList;
public QueueAclInfo() {
// JAXB needs this
}
public QueueAclInfo(String accessType, String accessControlList) {
this.accessType = accessType;
this.accessControlList = accessControlList;
}
public String getAccessType() {
return accessType;
}
public String getAccessControlList() {
return accessControlList;
}
}

47
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclsInfo.java Normal file
View File

@ -0,0 +1,47 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import java.util.ArrayList;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
@XmlRootElement(name = "queueAcls")
@XmlAccessorType(XmlAccessType.FIELD)
public class QueueAclsInfo {
protected ArrayList<QueueAclInfo> queueAcl = new ArrayList<QueueAclInfo>();
public QueueAclsInfo() {
} // JAXB needs this
public void add(QueueAclInfo queueAclInfo) {
queueAcl.add(queueAclInfo);
}
public ArrayList<QueueAclInfo> getQueueAcls() {
return queueAcl;
}
public void addAll(ArrayList<QueueAclInfo> queueAclsInfo) {
queueAcl.addAll(queueAclsInfo);
}
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
View File

@ -364,10 +364,10 @@ private void verifyClusterSchedulerGeneric(String type, float usedCapacity,
private void verifySubQueue(JSONObject info, String q,
float parentAbsCapacity, float parentAbsMaxCapacity)
throws JSONException, Exception {
int numExpectedElements = 20;
int numExpectedElements = 24;
boolean isParentQueue = true;
if (!info.has("queues")) {
numExpectedElements = 35;
numExpectedElements = 42;
isParentQueue = false;
}
assertEquals("incorrect number of elements", numExpectedElements, info.length());