YARN-10343. Legacy RM UI should include labeled metrics for allocated, total, and reserved resources. Contributed by Eric Payne
(cherry picked from commit 3eaf62726f
)
This commit is contained in:
parent
9263b75b3c
commit
d3c7f7cdbd
@ -200,6 +200,10 @@ public Resource getAllUsed() {
|
|||||||
return _getAll(ResourceType.USED);
|
return _getAll(ResourceType.USED);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Resource getAllReserved() {
|
||||||
|
return _getAll(ResourceType.RESERVED);
|
||||||
|
}
|
||||||
|
|
||||||
// Cache Used
|
// Cache Used
|
||||||
public Resource getCachedUsed() {
|
public Resource getCachedUsed() {
|
||||||
return _get(NL, ResourceType.CACHED_USED);
|
return _get(NL, ResourceType.CACHED_USED);
|
||||||
|
@ -22,6 +22,7 @@
|
|||||||
import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
|
import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
|
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
|
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo;
|
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.UserMetricsInfo;
|
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.UserMetricsInfo;
|
||||||
|
|
||||||
@ -61,6 +62,37 @@ protected void render(Block html) {
|
|||||||
|
|
||||||
DIV<Hamlet> div = html.div().$class("metrics");
|
DIV<Hamlet> div = html.div().$class("metrics");
|
||||||
|
|
||||||
|
long usedMemoryBytes = 0;
|
||||||
|
long totalMemoryBytes = 0;
|
||||||
|
long reservedMemoryBytes = 0;
|
||||||
|
long usedVCores = 0;
|
||||||
|
long totalVCores = 0;
|
||||||
|
long reservedVCores = 0;
|
||||||
|
if (clusterMetrics.getCrossPartitionMetricsAvailable()) {
|
||||||
|
ResourceInfo usedAllPartitions =
|
||||||
|
clusterMetrics.getTotalUsedResourcesAcrossPartition();
|
||||||
|
ResourceInfo totalAllPartitions =
|
||||||
|
clusterMetrics.getTotalClusterResourcesAcrossPartition();
|
||||||
|
ResourceInfo reservedAllPartitions =
|
||||||
|
clusterMetrics.getTotalReservedResourcesAcrossPartition();
|
||||||
|
usedMemoryBytes = usedAllPartitions.getMemorySize() * BYTES_IN_MB;
|
||||||
|
totalMemoryBytes = totalAllPartitions.getMemorySize() * BYTES_IN_MB;
|
||||||
|
reservedMemoryBytes = reservedAllPartitions.getMemorySize() * BYTES_IN_MB;
|
||||||
|
usedVCores = usedAllPartitions.getvCores();
|
||||||
|
totalVCores = totalAllPartitions.getvCores();
|
||||||
|
reservedVCores = reservedAllPartitions.getvCores();
|
||||||
|
// getTotalUsedResourcesAcrossPartition includes reserved resources.
|
||||||
|
usedMemoryBytes -= reservedMemoryBytes;
|
||||||
|
usedVCores -= reservedVCores;
|
||||||
|
} else {
|
||||||
|
usedMemoryBytes = clusterMetrics.getAllocatedMB() * BYTES_IN_MB;
|
||||||
|
totalMemoryBytes = clusterMetrics.getTotalMB() * BYTES_IN_MB;
|
||||||
|
reservedMemoryBytes = clusterMetrics.getReservedMB() * BYTES_IN_MB;
|
||||||
|
usedVCores = clusterMetrics.getAllocatedVirtualCores();
|
||||||
|
totalVCores = clusterMetrics.getTotalVirtualCores();
|
||||||
|
reservedVCores = clusterMetrics.getReservedVirtualCores();
|
||||||
|
}
|
||||||
|
|
||||||
div.h3("Cluster Metrics").
|
div.h3("Cluster Metrics").
|
||||||
table("#metricsoverview").
|
table("#metricsoverview").
|
||||||
thead().$class("ui-widget-header").
|
thead().$class("ui-widget-header").
|
||||||
@ -89,13 +121,14 @@ protected void render(Block html) {
|
|||||||
clusterMetrics.getAppsFailed() + clusterMetrics.getAppsKilled()
|
clusterMetrics.getAppsFailed() + clusterMetrics.getAppsKilled()
|
||||||
)
|
)
|
||||||
).
|
).
|
||||||
td(String.valueOf(clusterMetrics.getContainersAllocated())).
|
td(String.valueOf(
|
||||||
td(StringUtils.byteDesc(clusterMetrics.getAllocatedMB() * BYTES_IN_MB)).
|
clusterMetrics.getTotalAllocatedContainersAcrossPartition())).
|
||||||
td(StringUtils.byteDesc(clusterMetrics.getTotalMB() * BYTES_IN_MB)).
|
td(StringUtils.byteDesc(usedMemoryBytes)).
|
||||||
td(StringUtils.byteDesc(clusterMetrics.getReservedMB() * BYTES_IN_MB)).
|
td(StringUtils.byteDesc(totalMemoryBytes)).
|
||||||
td(String.valueOf(clusterMetrics.getAllocatedVirtualCores())).
|
td(StringUtils.byteDesc(reservedMemoryBytes)).
|
||||||
td(String.valueOf(clusterMetrics.getTotalVirtualCores())).
|
td(String.valueOf(usedVCores)).
|
||||||
td(String.valueOf(clusterMetrics.getReservedVirtualCores())).
|
td(String.valueOf(totalVCores)).
|
||||||
|
td(String.valueOf(reservedVCores)).
|
||||||
__().
|
__().
|
||||||
__().__();
|
__().__();
|
||||||
|
|
||||||
|
@ -26,6 +26,7 @@
|
|||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
|
||||||
|
|
||||||
@XmlRootElement(name = "clusterMetrics")
|
@XmlRootElement(name = "clusterMetrics")
|
||||||
@XmlAccessorType(XmlAccessType.FIELD)
|
@XmlAccessorType(XmlAccessType.FIELD)
|
||||||
@ -69,6 +70,14 @@ public class ClusterMetricsInfo {
|
|||||||
// Total registered resources of the cluster, including all partitions
|
// Total registered resources of the cluster, including all partitions
|
||||||
private ResourceInfo totalClusterResourcesAcrossPartition;
|
private ResourceInfo totalClusterResourcesAcrossPartition;
|
||||||
|
|
||||||
|
// Total reserved resources of the cluster, including all partitions.
|
||||||
|
private ResourceInfo totalReservedResourcesAcrossPartition;
|
||||||
|
|
||||||
|
// Total allocated containers across all partitions.
|
||||||
|
private int totalAllocatedContainersAcrossPartition;
|
||||||
|
|
||||||
|
private boolean crossPartitionMetricsAvailable = false;
|
||||||
|
|
||||||
public ClusterMetricsInfo() {
|
public ClusterMetricsInfo() {
|
||||||
} // JAXB needs this
|
} // JAXB needs this
|
||||||
|
|
||||||
@ -115,6 +124,11 @@ public ClusterMetricsInfo(final ResourceScheduler rs) {
|
|||||||
cs.getRootQueue().getQueueResourceUsage().getAllUsed());
|
cs.getRootQueue().getQueueResourceUsage().getAllUsed());
|
||||||
totalClusterResourcesAcrossPartition = new ResourceInfo(
|
totalClusterResourcesAcrossPartition = new ResourceInfo(
|
||||||
cs.getClusterResource());
|
cs.getClusterResource());
|
||||||
|
totalReservedResourcesAcrossPartition = new ResourceInfo(
|
||||||
|
cs.getRootQueue().getQueueResourceUsage().getAllReserved());
|
||||||
|
totalAllocatedContainersAcrossPartition =
|
||||||
|
((ParentQueue) cs.getRootQueue()).getNumContainers();
|
||||||
|
crossPartitionMetricsAvailable = true;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
this.totalMB = availableMB + allocatedMB;
|
this.totalMB = availableMB + allocatedMB;
|
||||||
@ -346,4 +360,16 @@ public ResourceInfo getTotalUsedResourcesAcrossPartition() {
|
|||||||
public ResourceInfo getTotalClusterResourcesAcrossPartition() {
|
public ResourceInfo getTotalClusterResourcesAcrossPartition() {
|
||||||
return totalClusterResourcesAcrossPartition;
|
return totalClusterResourcesAcrossPartition;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public ResourceInfo getTotalReservedResourcesAcrossPartition() {
|
||||||
|
return totalReservedResourcesAcrossPartition;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getTotalAllocatedContainersAcrossPartition() {
|
||||||
|
return totalAllocatedContainersAcrossPartition;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean getCrossPartitionMetricsAvailable() {
|
||||||
|
return crossPartitionMetricsAvailable;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -474,7 +474,7 @@ public void verifyClusterMetricsJSON(JSONObject json) throws JSONException,
|
|||||||
Exception {
|
Exception {
|
||||||
assertEquals("incorrect number of elements", 1, json.length());
|
assertEquals("incorrect number of elements", 1, json.length());
|
||||||
JSONObject clusterinfo = json.getJSONObject("clusterMetrics");
|
JSONObject clusterinfo = json.getJSONObject("clusterMetrics");
|
||||||
assertEquals("incorrect number of elements", 27, clusterinfo.length());
|
assertEquals("incorrect number of elements", 29, clusterinfo.length());
|
||||||
verifyClusterMetrics(
|
verifyClusterMetrics(
|
||||||
clusterinfo.getInt("appsSubmitted"), clusterinfo.getInt("appsCompleted"),
|
clusterinfo.getInt("appsSubmitted"), clusterinfo.getInt("appsCompleted"),
|
||||||
clusterinfo.getInt("reservedMB"), clusterinfo.getInt("availableMB"),
|
clusterinfo.getInt("reservedMB"), clusterinfo.getInt("availableMB"),
|
||||||
|
Loading…
Reference in New Issue
Block a user