big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-7817. Add Resource reference to RM's NodeInfo object so REST API can get non memory/vcore resource usages. (Sunil G via wangda)

Browse Source 
Change-Id: Ia7ceeabd82046645ddeaf487c763288f36cfbdee
This commit is contained in:
Wangda Tan 2018-01-26 15:43:27 +08:00
parent 2e5865606b
commit e0cfb0a31a
8 changed files with 114 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
View File

@ -55,7 +55,7 @@ public JAXBContextResolver() throws Exception {
UsersInfo.class, UserInfo.class, ApplicationStatisticsInfo.class,
StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class,
FairSchedulerQueueInfoList.class, AppTimeoutsInfo.class,
AppTimeoutInfo.class };
AppTimeoutInfo.class, ResourceInformationsInfo.class };
// these dao classes need root unwrapping
final Class[] rootUnwrappedTypes =
{ NewApplication.class, ApplicationSubmissionContextInfo.class,

24
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
View File

@ -61,6 +61,12 @@ public class ClusterMetricsInfo {
private int activeNodes;
private int shutdownNodes;
// Total used resource of the cluster, including all partitions
private ResourceInfo totalUsedResourcesAcrossPartition;
// Total registered resources of the cluster, including all partitions
private ResourceInfo totalClusterResourcesAcrossPartition;
public ClusterMetricsInfo() {
} // JAXB needs this
@ -92,9 +98,20 @@ public ClusterMetricsInfo(final ResourceScheduler rs) {
this.containersReserved = metrics.getReservedContainers();
if (rs instanceof CapacityScheduler) {
CapacityScheduler cs = (CapacityScheduler) rs;
this.totalMB = availableMB + allocatedMB + reservedMB;
this.totalVirtualCores =
availableVirtualCores + allocatedVirtualCores + containersReserved;
// TODO, add support of other schedulers to get total used resources
// across partition.
if (cs.getRootQueue() != null
&& cs.getRootQueue().getQueueResourceUsage() != null
&& cs.getRootQueue().getQueueResourceUsage().getAllUsed() != null) {
totalUsedResourcesAcrossPartition = new ResourceInfo(
cs.getRootQueue().getQueueResourceUsage().getAllUsed());
totalClusterResourcesAcrossPartition = new ResourceInfo(
cs.getClusterResource());
}
} else {
this.totalMB = availableMB + allocatedMB;
this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
@ -310,4 +327,11 @@ public void setShutdownNodes(int shutdownNodes) {
this.shutdownNodes = shutdownNodes;
}
public ResourceInfo getTotalUsedResourcesAcrossPartition() {
return totalUsedResourcesAcrossPartition;
}
public ResourceInfo getTotalClusterResourcesAcrossPartition() {
return totalClusterResourcesAcrossPartition;
}
}

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
View File

@ -58,6 +58,8 @@ public class NodeInfo {
private int numQueuedContainers;
protected ArrayList<String> nodeLabels = new ArrayList<String>();
protected ResourceUtilizationInfo resourceUtilization;
protected ResourceInfo usedResource;
protected ResourceInfo availableResource;
public NodeInfo() {
} // JAXB needs this
@ -75,6 +77,8 @@ public NodeInfo(RMNode ni, ResourceScheduler sched) {
this.usedVirtualCores = report.getUsedResource().getVirtualCores();
this.availableVirtualCores =
report.getAvailableResource().getVirtualCores();
this.usedResource = new ResourceInfo(report.getUsedResource());
this.availableResource = new ResourceInfo(report.getAvailableResource());
}
this.id = id.toString();
this.rack = ni.getRackName();
@ -183,6 +187,22 @@ public ArrayList<String> getNodeLabels() {
return this.nodeLabels;
}
public ResourceInfo getUsedResource() {
return usedResource;
}
public void setUsedResource(ResourceInfo used) {
this.usedResource = used;
}
public ResourceInfo getAvailableResource() {
return availableResource;
}
public void setAvailableResource(ResourceInfo avail) {
this.availableResource = avail;
}
public ResourceUtilizationInfo getResourceUtilization() {
return this.resourceUtilization;
}

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
View File

@ -26,6 +26,7 @@
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.util.resource.Resources;
@XmlRootElement
@XmlAccessorType(XmlAccessType.NONE)
public class ResourceInfo {
@ -34,6 +35,9 @@ public class ResourceInfo {
long memory;
@XmlElement
int vCores;
@XmlElement
ResourceInformationsInfo resourceInformations =
new ResourceInformationsInfo();
private Resource resources;
@ -41,9 +45,13 @@ public ResourceInfo() {
}
public ResourceInfo(Resource res) {
memory = res.getMemorySize();
vCores = res.getVirtualCores();
resources = Resources.clone(res);
// Make sure no NPE.
if (res != null) {
memory = res.getMemorySize();
vCores = res.getVirtualCores();
resources = Resources.clone(res);
resourceInformations.addAll(res.getAllResourcesListCopy());
}
}
public long getMemorySize() {
@ -84,4 +92,8 @@ public void setvCores(int vCores) {
public Resource getResource() {
return Resource.newInstance(resources);
}
public ResourceInformationsInfo getResourcesInformations() {
return resourceInformations;
}
}

48
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInformationsInfo.java Normal file
View File

@ -0,0 +1,48 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import java.util.ArrayList;
import java.util.List;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
@XmlRootElement(name = "resourceInformations")
@XmlAccessorType(XmlAccessType.FIELD)
public class ResourceInformationsInfo {
@XmlElement(name = "resourceInformation")
protected ArrayList<ResourceInformation> resourceInformation =
new ArrayList<ResourceInformation>();
public ResourceInformationsInfo() {
} // JAXB needs this
public ArrayList<ResourceInformation> getApps() {
return resourceInformation;
}
public void addAll(List<ResourceInformation> resourcesInformationsInfo) {
resourceInformation.addAll(resourcesInformationsInfo);
}
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js
View File

@ -169,8 +169,8 @@ export default DS.Model.extend({
&& this.get("totalUsedResourcesAcrossPartition")) {
var usages = [];
var clusterResourceInformations = this.get("totalClusterResourcesAcrossPartition").resourcesInformations;
var usedResourceInformations = this.get("totalUsedResourcesAcrossPartition").resourcesInformations;
var clusterResourceInformations = this.get("totalClusterResourcesAcrossPartition").resourceInformations.resourceInformation;
var usedResourceInformations = this.get("totalUsedResourcesAcrossPartition").resourceInformations.resourceInformation;
clusterResourceInformations.forEach(function(cluster) {
var perResourceTypeUsage = {

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js
View File

@ -99,7 +99,7 @@ export default DS.Model.extend({
const usedResource = this.get("usedResource");
const availableResource = this.get("availableResource");
var resourceInformations = usedResource ? usedResource.resourcesInformations : [];
var resourceInformations = usedResource ? usedResource.resourceInformations.resourceInformation : [];
for (var i = 0; i < resourceInformations.length; i++) {
ri = resourceInformations[i];
if (ri.name === "yarn.io/gpu") {
@ -108,7 +108,7 @@ export default DS.Model.extend({
}
var available = 0;
resourceInformations = availableResource ? availableResource.resourcesInformations : [];
resourceInformations = availableResource ? availableResource.resourceInformations.resourceInformation : [];
for (i = 0; i < resourceInformations.length; i++) {
ri = resourceInformations[i];
if (ri.name === "yarn.io/gpu") {

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js
View File

@ -42,8 +42,8 @@ export default DS.JSONAPISerializer.extend({
availableVirtualCores: payload.availableVirtualCores,
version: payload.version,
nodeLabels: payload.nodeLabels,
usedResource: payload.used,
availableResource: payload.avail
usedResource: payload.usedResource,
availableResource: payload.availableResource
}
};
return fixedPayload;