From 13a5803ccf9c55acf2a8f6c0d484dd2ed56e86d3 Mon Sep 17 00:00:00 2001 From: Abhishek Modi Date: Mon, 12 Aug 2019 14:31:24 +0530 Subject: [PATCH] YARN-9464. Support pending resource metrics in RM's RESTful API. Contributed by Prabhu Joseph. --- .../webapp/dao/ClusterMetricsInfo.java | 12 +++++++++ .../webapp/TestRMWebServices.java | 27 ++++++++++++++----- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java index 69d88aacca..d6e4828fbc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java @@ -41,10 +41,12 @@ public class ClusterMetricsInfo { private long reservedMB; private long availableMB; private long allocatedMB; + private long pendingMB; private long reservedVirtualCores; private long availableVirtualCores; private long allocatedVirtualCores; + private long pendingVirtualCores; private int containersAllocated; private int containersReserved; @@ -88,10 +90,12 @@ public ClusterMetricsInfo(final ResourceScheduler rs) { this.reservedMB = metrics.getReservedMB(); this.availableMB = metrics.getAvailableMB(); this.allocatedMB = metrics.getAllocatedMB(); + this.pendingMB = metrics.getPendingMB(); this.reservedVirtualCores = metrics.getReservedVirtualCores(); this.availableVirtualCores = metrics.getAvailableVirtualCores(); this.allocatedVirtualCores = metrics.getAllocatedVirtualCores(); + this.pendingVirtualCores = metrics.getPendingVirtualCores(); this.containersAllocated = metrics.getAllocatedContainers(); this.containersPending = metrics.getPendingContainers(); @@ -163,6 +167,10 @@ public long getAllocatedMB() { return this.allocatedMB; } + public long getPendingMB() { + return this.pendingMB; + } + public long getReservedVirtualCores() { return this.reservedVirtualCores; } @@ -175,6 +183,10 @@ public long getAllocatedVirtualCores() { return this.allocatedVirtualCores; } + public long getPendingVirtualCores() { + return this.pendingVirtualCores; + } + public int getContainersAllocated() { return this.containersAllocated; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java index cb8056624a..9e9fe6e60d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java @@ -426,9 +426,11 @@ public void verifyClusterMetricsXML(String xml) throws JSONException, WebServicesTestUtils.getXmlInt(element, "reservedMB"), WebServicesTestUtils.getXmlInt(element, "availableMB"), WebServicesTestUtils.getXmlInt(element, "allocatedMB"), + WebServicesTestUtils.getXmlInt(element, "pendingMB"), WebServicesTestUtils.getXmlInt(element, "reservedVirtualCores"), WebServicesTestUtils.getXmlInt(element, "availableVirtualCores"), WebServicesTestUtils.getXmlInt(element, "allocatedVirtualCores"), + WebServicesTestUtils.getXmlInt(element, "pendingVirtualCores"), WebServicesTestUtils.getXmlInt(element, "totalVirtualCores"), WebServicesTestUtils.getXmlInt(element, "containersAllocated"), WebServicesTestUtils.getXmlInt(element, "totalMB"), @@ -446,13 +448,16 @@ public void verifyClusterMetricsJSON(JSONObject json) throws JSONException, Exception { assertEquals("incorrect number of elements", 1, json.length()); JSONObject clusterinfo = json.getJSONObject("clusterMetrics"); - assertEquals("incorrect number of elements", 25, clusterinfo.length()); + assertEquals("incorrect number of elements", 27, clusterinfo.length()); verifyClusterMetrics( clusterinfo.getInt("appsSubmitted"), clusterinfo.getInt("appsCompleted"), clusterinfo.getInt("reservedMB"), clusterinfo.getInt("availableMB"), - clusterinfo.getInt("allocatedMB"), - clusterinfo.getInt("reservedVirtualCores"), clusterinfo.getInt("availableVirtualCores"), - clusterinfo.getInt("allocatedVirtualCores"), clusterinfo.getInt("totalVirtualCores"), + clusterinfo.getInt("allocatedMB"), clusterinfo.getInt("pendingMB"), + clusterinfo.getInt("reservedVirtualCores"), + clusterinfo.getInt("availableVirtualCores"), + clusterinfo.getInt("allocatedVirtualCores"), + clusterinfo.getInt("pendingVirtualCores"), + clusterinfo.getInt("totalVirtualCores"), clusterinfo.getInt("containersAllocated"), clusterinfo.getInt("totalMB"), clusterinfo.getInt("totalNodes"), clusterinfo.getInt("lostNodes"), clusterinfo.getInt("unhealthyNodes"), @@ -462,8 +467,9 @@ public void verifyClusterMetricsJSON(JSONObject json) throws JSONException, } public void verifyClusterMetrics(int submittedApps, int completedApps, - int reservedMB, int availableMB, int allocMB, int reservedVirtualCores, - int availableVirtualCores, int allocVirtualCores, int totalVirtualCores, + int reservedMB, int availableMB, int allocMB, int pendingMB, + int reservedVirtualCores, int availableVirtualCores, + int allocVirtualCores, int pendingVirtualCores, int totalVirtualCores, int containersAlloc, int totalMB, int totalNodes, int lostNodes, int unhealthyNodes, int decommissionedNodes, int rebootedNodes, int activeNodes, int shutdownNodes) throws JSONException, Exception { @@ -486,12 +492,19 @@ public void verifyClusterMetrics(int submittedApps, int completedApps, metrics.getAvailableMB(), availableMB); assertEquals("allocatedMB doesn't match", metrics.getAllocatedMB(), allocMB); + assertEquals("pendingMB doesn't match", + metrics.getPendingMB(), pendingMB); assertEquals("reservedVirtualCores doesn't match", metrics.getReservedVirtualCores(), reservedVirtualCores); assertEquals("availableVirtualCores doesn't match", metrics.getAvailableVirtualCores(), availableVirtualCores); + assertEquals("pendingVirtualCores doesn't match", + metrics.getPendingVirtualCores(), pendingVirtualCores); assertEquals("allocatedVirtualCores doesn't match", - totalVirtualCoresExpect, allocVirtualCores); + metrics.getAllocatedVirtualCores(), allocVirtualCores); + assertEquals("totalVirtualCores doesn't match", + totalVirtualCoresExpect, totalVirtualCores); + assertEquals("containersAllocated doesn't match", 0, containersAlloc); assertEquals("totalMB doesn't match", totalMBExpect, totalMB); assertEquals(