From d3c7f7cdbdaf98837f273ec561569d64c5c6efe7 Mon Sep 17 00:00:00 2001 From: Jonathan Hung Date: Tue, 28 Jul 2020 13:43:19 -0700 Subject: [PATCH] YARN-10343. Legacy RM UI should include labeled metrics for allocated, total, and reserved resources. Contributed by Eric Payne (cherry picked from commit 3eaf62726ffe90b3b096798fe501abd1ed0c5f15) --- .../scheduler/ResourceUsage.java | 4 ++ .../webapp/MetricsOverviewTable.java | 49 ++++++++++++++++--- .../webapp/dao/ClusterMetricsInfo.java | 26 ++++++++++ .../webapp/TestRMWebServices.java | 2 +- 4 files changed, 72 insertions(+), 9 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java index c46c9112ea..18fd6c3567 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java @@ -200,6 +200,10 @@ public Resource getAllUsed() { return _getAll(ResourceType.USED); } + public Resource getAllReserved() { + return _getAll(ResourceType.RESERVED); + } + // Cache Used public Resource getCachedUsed() { return _get(NL, ResourceType.CACHED_USED); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java index 806b636409..fbaeafd921 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java @@ -22,6 +22,7 @@ import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.UserMetricsInfo; @@ -60,7 +61,38 @@ protected void render(Block html) { ClusterMetricsInfo clusterMetrics = new ClusterMetricsInfo(this.rm); DIV div = html.div().$class("metrics"); - + + long usedMemoryBytes = 0; + long totalMemoryBytes = 0; + long reservedMemoryBytes = 0; + long usedVCores = 0; + long totalVCores = 0; + long reservedVCores = 0; + if (clusterMetrics.getCrossPartitionMetricsAvailable()) { + ResourceInfo usedAllPartitions = + clusterMetrics.getTotalUsedResourcesAcrossPartition(); + ResourceInfo totalAllPartitions = + clusterMetrics.getTotalClusterResourcesAcrossPartition(); + ResourceInfo reservedAllPartitions = + clusterMetrics.getTotalReservedResourcesAcrossPartition(); + usedMemoryBytes = usedAllPartitions.getMemorySize() * BYTES_IN_MB; + totalMemoryBytes = totalAllPartitions.getMemorySize() * BYTES_IN_MB; + reservedMemoryBytes = reservedAllPartitions.getMemorySize() * BYTES_IN_MB; + usedVCores = usedAllPartitions.getvCores(); + totalVCores = totalAllPartitions.getvCores(); + reservedVCores = reservedAllPartitions.getvCores(); + // getTotalUsedResourcesAcrossPartition includes reserved resources. + usedMemoryBytes -= reservedMemoryBytes; + usedVCores -= reservedVCores; + } else { + usedMemoryBytes = clusterMetrics.getAllocatedMB() * BYTES_IN_MB; + totalMemoryBytes = clusterMetrics.getTotalMB() * BYTES_IN_MB; + reservedMemoryBytes = clusterMetrics.getReservedMB() * BYTES_IN_MB; + usedVCores = clusterMetrics.getAllocatedVirtualCores(); + totalVCores = clusterMetrics.getTotalVirtualCores(); + reservedVCores = clusterMetrics.getReservedVirtualCores(); + } + div.h3("Cluster Metrics"). table("#metricsoverview"). thead().$class("ui-widget-header"). @@ -89,13 +121,14 @@ protected void render(Block html) { clusterMetrics.getAppsFailed() + clusterMetrics.getAppsKilled() ) ). - td(String.valueOf(clusterMetrics.getContainersAllocated())). - td(StringUtils.byteDesc(clusterMetrics.getAllocatedMB() * BYTES_IN_MB)). - td(StringUtils.byteDesc(clusterMetrics.getTotalMB() * BYTES_IN_MB)). - td(StringUtils.byteDesc(clusterMetrics.getReservedMB() * BYTES_IN_MB)). - td(String.valueOf(clusterMetrics.getAllocatedVirtualCores())). - td(String.valueOf(clusterMetrics.getTotalVirtualCores())). - td(String.valueOf(clusterMetrics.getReservedVirtualCores())). + td(String.valueOf( + clusterMetrics.getTotalAllocatedContainersAcrossPartition())). + td(StringUtils.byteDesc(usedMemoryBytes)). + td(StringUtils.byteDesc(totalMemoryBytes)). + td(StringUtils.byteDesc(reservedMemoryBytes)). + td(String.valueOf(usedVCores)). + td(String.valueOf(totalVCores)). + td(String.valueOf(reservedVCores)). __(). __().__(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java index d6e4828fbc..ee76eeacad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java @@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue; @XmlRootElement(name = "clusterMetrics") @XmlAccessorType(XmlAccessType.FIELD) @@ -69,6 +70,14 @@ public class ClusterMetricsInfo { // Total registered resources of the cluster, including all partitions private ResourceInfo totalClusterResourcesAcrossPartition; + // Total reserved resources of the cluster, including all partitions. + private ResourceInfo totalReservedResourcesAcrossPartition; + + // Total allocated containers across all partitions. + private int totalAllocatedContainersAcrossPartition; + + private boolean crossPartitionMetricsAvailable = false; + public ClusterMetricsInfo() { } // JAXB needs this @@ -115,6 +124,11 @@ public ClusterMetricsInfo(final ResourceScheduler rs) { cs.getRootQueue().getQueueResourceUsage().getAllUsed()); totalClusterResourcesAcrossPartition = new ResourceInfo( cs.getClusterResource()); + totalReservedResourcesAcrossPartition = new ResourceInfo( + cs.getRootQueue().getQueueResourceUsage().getAllReserved()); + totalAllocatedContainersAcrossPartition = + ((ParentQueue) cs.getRootQueue()).getNumContainers(); + crossPartitionMetricsAvailable = true; } } else { this.totalMB = availableMB + allocatedMB; @@ -346,4 +360,16 @@ public ResourceInfo getTotalUsedResourcesAcrossPartition() { public ResourceInfo getTotalClusterResourcesAcrossPartition() { return totalClusterResourcesAcrossPartition; } + + public ResourceInfo getTotalReservedResourcesAcrossPartition() { + return totalReservedResourcesAcrossPartition; + } + + public int getTotalAllocatedContainersAcrossPartition() { + return totalAllocatedContainersAcrossPartition; + } + + public boolean getCrossPartitionMetricsAvailable() { + return crossPartitionMetricsAvailable; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java index afc37fa57a..079737cc81 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java @@ -474,7 +474,7 @@ public void verifyClusterMetricsJSON(JSONObject json) throws JSONException, Exception { assertEquals("incorrect number of elements", 1, json.length()); JSONObject clusterinfo = json.getJSONObject("clusterMetrics"); - assertEquals("incorrect number of elements", 27, clusterinfo.length()); + assertEquals("incorrect number of elements", 29, clusterinfo.length()); verifyClusterMetrics( clusterinfo.getInt("appsSubmitted"), clusterinfo.getInt("appsCompleted"), clusterinfo.getInt("reservedMB"), clusterinfo.getInt("availableMB"),