From c7007a8a3508a24a6f023302516fe8fbe270eab1 Mon Sep 17 00:00:00 2001 From: Arun Murthy Date: Mon, 6 Feb 2012 03:38:44 +0000 Subject: [PATCH] MAPREDUCE-3747. Initialize queue metrics upfront and added start/finish time to RM Web-UI. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1240886 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../scheduler/capacity/CSQueue.java | 12 ++++ .../scheduler/capacity/CSQueueUtils.java | 38 +++++++++++ .../scheduler/capacity/LeafQueue.java | 66 +++++++++---------- .../scheduler/capacity/ParentQueue.java | 51 +++++++------- .../resourcemanager/webapp/AppsBlock.java | 16 +++-- .../webapp/MetricsOverviewTable.java | 49 ++++++++++---- .../server/resourcemanager/webapp/RmView.java | 7 +- .../webapp/dao/ClusterMetricsInfo.java | 48 ++++++++++++++ .../webapp/dao/UserMetricsInfo.java | 33 ++++++++++ .../capacity/TestApplicationLimits.java | 10 ++- .../resourcemanager/webapp/TestNodesPage.java | 2 +- .../webapp/TestRMWebServices.java | 13 ++-- 13 files changed, 260 insertions(+), 88 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 12b9ce0bc0..742adbcce0 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -705,6 +705,9 @@ Release 0.23.1 - Unreleased MAPREDUCE-3765. FifoScheduler does not respect yarn.scheduler.fifo.minimum- allocation-mb setting (Hitesh Shah via mahadev) + MAPREDUCE-3747. Initialize queue metrics upfront and added start/finish + time to RM Web-UI. (acmurthy) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java index b646e14fb8..0730cfc25d 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java @@ -98,6 +98,12 @@ public interface CSQueue */ public float getUsedCapacity(); + /** + * Set used capacity of the queue. + * @param usedCapacity used capacity of the queue + */ + public void setUsedCapacity(float usedCapacity); + /** * Get the currently utilized resources in the cluster * by the queue and children (if any). @@ -114,6 +120,12 @@ public interface CSQueue */ public float getUtilization(); + /** + * Get the current utilization of the queue. + * @param utilization queue utilization + */ + public void setUtilization(float utilization); + /** * Get the current run-state of the queue * @return current run-state diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java index 89c36ab87a..01f14ebc53 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java @@ -17,7 +17,9 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; +import org.apache.hadoop.yarn.Lock; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; class CSQueueUtils { @@ -65,4 +67,40 @@ public static int computeMaxActiveApplicationsPerUser( 1); } + @Lock(CSQueue.class) + public static void updateQueueStatistics( + final CSQueue childQueue, final CSQueue parentQueue, + final Resource clusterResource, final Resource minimumAllocation) { + final int clusterMemory = clusterResource.getMemory(); + final int usedMemory = childQueue.getUsedResources().getMemory(); + + float queueLimit = 0.0f; + float utilization = 0.0f; + float usedCapacity = 0.0f; + if (clusterMemory > 0) { + queueLimit = clusterMemory * childQueue.getAbsoluteCapacity(); + final float parentAbsoluteCapacity = + (parentQueue == null) ? 1.0f : parentQueue.getAbsoluteCapacity(); + utilization = (usedMemory / queueLimit); + usedCapacity = (usedMemory / (clusterMemory * parentAbsoluteCapacity)); + } + + childQueue.setUtilization(utilization); + childQueue.setUsedCapacity(usedCapacity); + + int available = + Math.max((roundUp(minimumAllocation, (int)queueLimit) - usedMemory), 0); + childQueue.getMetrics().setAvailableResourcesToQueue( + Resources.createResource(available)); + } + + public static int roundUp(Resource minimumAllocation, int memory) { + int minMemory = minimumAllocation.getMemory(); + return LeafQueue.divideAndCeil(memory, minMemory) * minMemory; + } + + public static int roundDown(Resource minimumAllocation, int memory) { + int minMemory = minimumAllocation.getMemory(); + return (memory / minMemory) * minMemory; + } } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index ecc672c1cb..f21cfc21fc 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -180,7 +180,9 @@ public LeafQueue(CapacitySchedulerContext cs, Map acls = cs.getConfiguration().getAcls(getQueuePath()); - setupQueueConfigs(capacity, absoluteCapacity, + setupQueueConfigs( + cs.getClusterResources(), + capacity, absoluteCapacity, maximumCapacity, absoluteMaxCapacity, userLimit, userLimitFactor, maxApplications, maxApplicationsPerUser, @@ -198,6 +200,7 @@ public LeafQueue(CapacitySchedulerContext cs, } private synchronized void setupQueueConfigs( + Resource clusterResource, float capacity, float absoluteCapacity, float maximumCapacity, float absoluteMaxCapacity, int userLimit, float userLimitFactor, @@ -235,6 +238,10 @@ private synchronized void setupQueueConfigs( for (Map.Entry e : acls.entrySet()) { aclsString.append(e.getKey() + ":" + e.getValue().getAclString()); } + + // Update metrics + CSQueueUtils.updateQueueStatistics( + this, parent, clusterResource, minimumAllocation); LOG.info("Initializing " + queueName + "\n" + "capacity = " + capacity + @@ -386,11 +393,11 @@ public List getChildQueues() { return null; } - synchronized void setUtilization(float utilization) { + public synchronized void setUtilization(float utilization) { this.utilization = utilization; } - synchronized void setUsedCapacity(float usedCapacity) { + public synchronized void setUsedCapacity(float usedCapacity) { this.usedCapacity = usedCapacity; } @@ -534,7 +541,9 @@ public synchronized void reinitialize(CSQueue queue, Resource clusterResource) } LeafQueue leafQueue = (LeafQueue)queue; - setupQueueConfigs(leafQueue.capacity, leafQueue.absoluteCapacity, + setupQueueConfigs( + clusterResource, + leafQueue.capacity, leafQueue.absoluteCapacity, leafQueue.maximumCapacity, leafQueue.absoluteMaxCapacity, leafQueue.userLimit, leafQueue.userLimitFactor, leafQueue.maxApplications, @@ -542,8 +551,6 @@ public synchronized void reinitialize(CSQueue queue, Resource clusterResource) leafQueue.getMaximumActiveApplications(), leafQueue.getMaximumActiveApplicationsPerUser(), leafQueue.state, leafQueue.acls); - - updateResource(clusterResource); } @Override @@ -883,7 +890,8 @@ private Resource computeUserLimitAndSetHeadroom( Resource queueMaxCap = // Queue Max-Capacity Resources.createResource( - roundDown((int)(absoluteMaxCapacity * clusterResource.getMemory())) + CSQueueUtils.roundDown(minimumAllocation, + (int)(absoluteMaxCapacity * clusterResource.getMemory())) ); Resource userConsumed = getUser(user).getConsumedResources(); @@ -904,16 +912,6 @@ private Resource computeUserLimitAndSetHeadroom( return userLimit; } - private int roundUp(int memory) { - int minMemory = minimumAllocation.getMemory(); - return divideAndCeil(memory, minMemory) * minMemory; - } - - private int roundDown(int memory) { - int minMemory = minimumAllocation.getMemory(); - return (memory / minMemory) * minMemory; - } - @Lock(NoLock.class) private Resource computeUserLimit(SchedulerApp application, Resource clusterResource, Resource required) { @@ -927,8 +925,11 @@ private Resource computeUserLimit(SchedulerApp application, // Allow progress for queues with miniscule capacity final int queueCapacity = Math.max( - roundUp((int)(absoluteCapacity * clusterResource.getMemory())), - required.getMemory()); + CSQueueUtils.roundUp( + minimumAllocation, + (int)(absoluteCapacity * clusterResource.getMemory())), + required.getMemory() + ); final int consumed = usedResources.getMemory(); final int currentCapacity = @@ -943,7 +944,8 @@ private Resource computeUserLimit(SchedulerApp application, final int activeUsers = activeUsersManager.getNumActiveUsers(); int limit = - roundUp( + CSQueueUtils.roundUp( + minimumAllocation, Math.min( Math.max(divideAndCeil(currentCapacity, activeUsers), divideAndCeil((int)userLimit*currentCapacity, 100)), @@ -991,7 +993,7 @@ private synchronized boolean assignToUser(String userName, Resource limit) { return true; } - private static int divideAndCeil(int a, int b) { + static int divideAndCeil(int a, int b) { if (b == 0) { LOG.info("divideAndCeil called with a=" + a + " b=" + b); return 0; @@ -1325,7 +1327,8 @@ synchronized void allocateResource(Resource clusterResource, SchedulerApp application, Resource resource) { // Update queue metrics Resources.addTo(usedResources, resource); - updateResource(clusterResource); + CSQueueUtils.updateQueueStatistics( + this, parent, clusterResource, minimumAllocation); ++numContainers; // Update user metrics @@ -1349,7 +1352,8 @@ synchronized void releaseResource(Resource clusterResource, SchedulerApp application, Resource resource) { // Update queue metrics Resources.subtractFrom(usedResources, resource); - updateResource(clusterResource); + CSQueueUtils.updateQueueStatistics( + this, parent, clusterResource, minimumAllocation); --numContainers; // Update user metrics @@ -1374,6 +1378,10 @@ public synchronized void updateClusterResource(Resource clusterResource) { CSQueueUtils.computeMaxActiveApplicationsPerUser( maxActiveApplications, userLimit, userLimitFactor); + // Update metrics + CSQueueUtils.updateQueueStatistics( + this, parent, clusterResource, minimumAllocation); + // Update application properties for (SchedulerApp application : activeApplications) { synchronized (application) { @@ -1383,18 +1391,6 @@ public synchronized void updateClusterResource(Resource clusterResource) { } } - private synchronized void updateResource(Resource clusterResource) { - float queueLimit = clusterResource.getMemory() * absoluteCapacity; - setUtilization(usedResources.getMemory() / queueLimit); - setUsedCapacity(usedResources.getMemory() - / (clusterResource.getMemory() * parent.getAbsoluteCapacity())); - - Resource resourceLimit = - Resources.createResource(roundUp((int)queueLimit)); - metrics.setAvailableResourcesToQueue( - Resources.subtractFrom(resourceLimit, usedResources)); - } - @Override public QueueMetrics getMetrics() { return metrics; diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index 39aa197f2b..4010aa0ce0 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -97,7 +97,8 @@ public class ParentQueue implements CSQueue { RecordFactoryProvider.getRecordFactory(null); public ParentQueue(CapacitySchedulerContext cs, - String queueName, Comparator comparator, CSQueue parent, CSQueue old) { + String queueName, Comparator comparator, + CSQueue parent, CSQueue old) { minimumAllocation = cs.getMinimumResourceCapability(); this.parent = parent; @@ -137,7 +138,8 @@ public ParentQueue(CapacitySchedulerContext cs, this.queueInfo.setQueueName(queueName); this.queueInfo.setChildQueues(new ArrayList()); - setupQueueConfigs(capacity, absoluteCapacity, + setupQueueConfigs(cs.getClusterResources(), + capacity, absoluteCapacity, maximumCapacity, absoluteMaxCapacity, state, acls); this.queueComparator = comparator; @@ -149,9 +151,10 @@ public ParentQueue(CapacitySchedulerContext cs, } private synchronized void setupQueueConfigs( - float capacity, float absoluteCapacity, - float maximumCapacity, float absoluteMaxCapacity, - QueueState state, Map acls + Resource clusterResource, + float capacity, float absoluteCapacity, + float maximumCapacity, float absoluteMaxCapacity, + QueueState state, Map acls ) { // Sanity check CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity); @@ -174,6 +177,10 @@ private synchronized void setupQueueConfigs( aclsString.append(e.getKey() + ":" + e.getValue().getAclString()); } + // Update metrics + CSQueueUtils.updateQueueStatistics( + this, parent, clusterResource, minimumAllocation); + LOG.info(queueName + ", capacity=" + capacity + ", asboluteCapacity=" + absoluteCapacity + @@ -384,12 +391,10 @@ public synchronized void reinitialize(CSQueue queue, Resource clusterResource) childQueues.addAll(currentChildQueues.values()); // Set new configs - setupQueueConfigs(parentQueue.capacity, parentQueue.absoluteCapacity, + setupQueueConfigs(clusterResource, + parentQueue.capacity, parentQueue.absoluteCapacity, parentQueue.maximumCapacity, parentQueue.absoluteMaxCapacity, parentQueue.state, parentQueue.acls); - - // Update - updateResource(clusterResource); } Map getQueues(Set queues) { @@ -485,11 +490,11 @@ public synchronized void removeApplication(SchedulerApp application, " #applications: " + getNumApplications()); } - synchronized void setUsedCapacity(float usedCapacity) { + public synchronized void setUsedCapacity(float usedCapacity) { this.usedCapacity = usedCapacity; } - synchronized void setUtilization(float utilization) { + public synchronized void setUtilization(float utilization) { this.utilization = utilization; } @@ -674,14 +679,16 @@ public void completedContainer(Resource clusterResource, synchronized void allocateResource(Resource clusterResource, Resource resource) { Resources.addTo(usedResources, resource); - updateResource(clusterResource); + CSQueueUtils.updateQueueStatistics( + this, parent, clusterResource, minimumAllocation); ++numContainers; } synchronized void releaseResource(Resource clusterResource, Resource resource) { Resources.subtractFrom(usedResources, resource); - updateResource(clusterResource); + CSQueueUtils.updateQueueStatistics( + this, parent, clusterResource, minimumAllocation); --numContainers; } @@ -691,22 +698,12 @@ public synchronized void updateClusterResource(Resource clusterResource) { for (CSQueue childQueue : childQueues) { childQueue.updateClusterResource(clusterResource); } + + // Update metrics + CSQueueUtils.updateQueueStatistics( + this, parent, clusterResource, minimumAllocation); } - private synchronized void updateResource(Resource clusterResource) { - float queueLimit = clusterResource.getMemory() * absoluteCapacity; - float parentAbsoluteCapacity = - (rootQueue) ? 1.0f : parent.getAbsoluteCapacity(); - setUtilization(usedResources.getMemory() / queueLimit); - setUsedCapacity(usedResources.getMemory() - / (clusterResource.getMemory() * parentAbsoluteCapacity)); - - Resource resourceLimit = - Resources.createResource((int)queueLimit); - metrics.setAvailableResourcesToQueue( - Resources.subtractFrom(resourceLimit, usedResources)); - } - @Override public QueueMetrics getMetrics() { return metrics; diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java index 9f4d0f9467..27deee34e8 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; import static org.apache.hadoop.yarn.util.StringHelper.join; -import static org.apache.hadoop.yarn.util.StringHelper.sjoin; import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE; import static org.apache.hadoop.yarn.webapp.view.JQueryUI._PROGRESSBAR; import static org.apache.hadoop.yarn.webapp.view.JQueryUI._PROGRESSBAR_VALUE; @@ -27,6 +26,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; +import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY; @@ -52,11 +52,12 @@ class AppsBlock extends HtmlBlock { th(".user", "User"). th(".name", "Name"). th(".queue", "Queue"). + th(".starttime", "StartTime"). + th(".finishtime", "FinishTime"). th(".state", "State"). th(".finalstatus", "FinalStatus"). th(".progress", "Progress"). - th(".ui", "Tracking UI"). - th(".note", "Note")._()._(). + th(".ui", "Tracking UI")._()._(). tbody(); int i = 0; String reqState = $(APP_STATE); @@ -67,6 +68,8 @@ class AppsBlock extends HtmlBlock { } AppInfo appInfo = new AppInfo(app, true); String percent = String.format("%.1f", appInfo.getProgress()); + String startTime = Times.format(appInfo.getStartTime()); + String finishTime = Times.format(appInfo.getFinishTime()); tbody. tr(). td(). @@ -75,6 +78,10 @@ class AppsBlock extends HtmlBlock { td(appInfo.getUser()). td(appInfo.getName()). td(appInfo.getQueue()). + td(). + br().$title(startTime)._()._(startTime)._(). + td(). + br().$title(startTime)._()._(finishTime)._(). td(appInfo.getState()). td(appInfo.getFinalStatus()). td(). @@ -85,8 +92,7 @@ class AppsBlock extends HtmlBlock { $style(join("width:", percent, '%'))._()._()._(). td(). a(!appInfo.isTrackingUrlReady()? - "#" : appInfo.getTrackingUrlPretty(), appInfo.getTrackingUI())._(). - td(appInfo.getNote())._(); + "#" : appInfo.getTrackingUrlPretty(), appInfo.getTrackingUI())._()._(); if (list.rendering != Render.HTML && ++i >= 20) break; } tbody._()._(); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java index 92a84a244c..7a8b681200 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java @@ -55,15 +55,19 @@ protected void render(Block html) { //CSS in the correct spot html.style(".metrics {margin-bottom:5px}"); - ClusterMetricsInfo clusterMetrics = new ClusterMetricsInfo(this.rm, this.rmContext); - + ClusterMetricsInfo clusterMetrics = + new ClusterMetricsInfo(this.rm, this.rmContext); DIV div = html.div().$class("metrics"); - div.table("#metricsoverview"). + div.h3("Cluster Metrics"). + table("#metricsoverview"). thead().$class("ui-widget-header"). tr(). th().$class("ui-state-default")._("Apps Submitted")._(). + th().$class("ui-state-default")._("Apps Pending")._(). + th().$class("ui-state-default")._("Apps Running")._(). + th().$class("ui-state-default")._("Apps Completed")._(). th().$class("ui-state-default")._("Containers Running")._(). th().$class("ui-state-default")._("Memory Used")._(). th().$class("ui-state-default")._("Memory Total")._(). @@ -78,6 +82,14 @@ protected void render(Block html) { tbody().$class("ui-widget-content"). tr(). td(String.valueOf(clusterMetrics.getAppsSubmitted())). + td(String.valueOf(clusterMetrics.getAppsPending())). + td(String.valueOf(clusterMetrics.getAppsRunning())). + td( + String.valueOf( + clusterMetrics.getAppsCompleted() + + clusterMetrics.getAppsFailed() + clusterMetrics.getAppsKilled() + ) + ). td(String.valueOf(clusterMetrics.getContainersAllocated())). td(StringUtils.byteDesc(clusterMetrics.getAllocatedMB() * BYTES_IN_MB)). td(StringUtils.byteDesc(clusterMetrics.getTotalMB() * BYTES_IN_MB)). @@ -89,26 +101,38 @@ protected void render(Block html) { td().a(url("nodes/rebooted"),String.valueOf(clusterMetrics.getRebootedNodes()))._(). _(). _()._(); - + String user = request().getRemoteUser(); if (user != null) { UserMetricsInfo userMetrics = new UserMetricsInfo(this.rm, this.rmContext, user); if (userMetrics.metricsAvailable()) { - div.table("#usermetricsoverview"). + div.h3("User Metrics for " + user). + table("#usermetricsoverview"). thead().$class("ui-widget-header"). tr(). - th().$class("ui-state-default")._("Apps Submitted ("+user+")")._(). - th().$class("ui-state-default")._("Containers Running ("+user+")")._(). - th().$class("ui-state-default")._("Containers Pending ("+user+")")._(). - th().$class("ui-state-default")._("Containers Reserved ("+user+")")._(). - th().$class("ui-state-default")._("Memory Used ("+user+")")._(). - th().$class("ui-state-default")._("Memory Pending ("+user+")")._(). - th().$class("ui-state-default")._("Memory Reserved ("+user+")")._(). + th().$class("ui-state-default")._("Apps Submitted")._(). + th().$class("ui-state-default")._("Apps Pending")._(). + th().$class("ui-state-default")._("Apps Running")._(). + th().$class("ui-state-default")._("Apps Completed")._(). + th().$class("ui-state-default")._("Containers Running")._(). + th().$class("ui-state-default")._("Containers Pending")._(). + th().$class("ui-state-default")._("Containers Reserved")._(). + th().$class("ui-state-default")._("Memory Used")._(). + th().$class("ui-state-default")._("Memory Pending")._(). + th().$class("ui-state-default")._("Memory Reserved")._(). _(). _(). tbody().$class("ui-widget-content"). tr(). td(String.valueOf(userMetrics.getAppsSubmitted())). + td(String.valueOf(userMetrics.getAppsPending())). + td(String.valueOf(userMetrics.getAppsRunning())). + td( + String.valueOf( + (userMetrics.getAppsCompleted() + + userMetrics.getAppsFailed() + userMetrics.getAppsKilled()) + ) + ). td(String.valueOf(userMetrics.getRunningContainers())). td(String.valueOf(userMetrics.getPendingContainers())). td(String.valueOf(userMetrics.getReservedContainers())). @@ -117,6 +141,7 @@ protected void render(Block html) { td(StringUtils.byteDesc(userMetrics.getReservedMB() * BYTES_IN_MB)). _(). _()._(); + } } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java index 47532c1562..bc2bfca328 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java @@ -63,10 +63,11 @@ protected Class content() { private String appsTableInit() { AppsList list = getInstance(AppsList.class); - // id, user, name, queue, state, progress, ui, note + // id, user, name, queue, starttime, finishtime, state, progress, ui StringBuilder init = tableInit(). - append(", aoColumns:[{sType:'title-numeric'}, null, null, null, null,"). - append("null,{sType:'title-numeric', bSearchable:false}, null, null]"); + append(", aoColumns:[{sType:'title-numeric'}, null, null, null, "). + append("null, null , null, "). + append("null,{sType:'title-numeric', bSearchable:false}, null]"); // Sort by id upon page load init.append(", aaSorting: [[0, 'asc']]"); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java index d8db4bb99b..0db26c2e08 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java @@ -32,10 +32,20 @@ public class ClusterMetricsInfo { protected int appsSubmitted; + protected int appsCompleted; + protected int appsPending; + protected int appsRunning; + protected int appsFailed; + protected int appsKilled; + protected long reservedMB; protected long availableMB; protected long allocatedMB; + protected int containersAllocated; + protected int containersReserved; + protected int containersPending; + protected long totalMB; protected int totalNodes; protected int lostNodes; @@ -53,10 +63,20 @@ public ClusterMetricsInfo(final ResourceManager rm, final RMContext rmContext) { ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics(); this.appsSubmitted = metrics.getAppsSubmitted(); + this.appsCompleted = metrics.getAppsCompleted(); + this.appsPending = metrics.getAppsPending(); + this.appsRunning = metrics.getAppsRunning(); + this.appsFailed = metrics.getAppsFailed(); + this.appsKilled = metrics.getAppsKilled(); + this.reservedMB = metrics.getReservedMB(); this.availableMB = metrics.getAvailableMB(); this.allocatedMB = metrics.getAllocatedMB(); + this.containersAllocated = metrics.getAllocatedContainers(); + this.containersPending = metrics.getPendingContainers(); + this.containersReserved = metrics.getReservedContainers(); + this.totalMB = availableMB + reservedMB + allocatedMB; this.activeNodes = clusterMetrics.getNumActiveNMs(); this.lostNodes = clusterMetrics.getNumLostNMs(); @@ -71,6 +91,26 @@ public int getAppsSubmitted() { return this.appsSubmitted; } + public int getAppsCompleted() { + return appsCompleted; + } + + public int getAppsPending() { + return appsPending; + } + + public int getAppsRunning() { + return appsRunning; + } + + public int getAppsFailed() { + return appsFailed; + } + + public int getAppsKilled() { + return appsKilled; + } + public long getReservedMB() { return this.reservedMB; } @@ -87,6 +127,14 @@ public int getContainersAllocated() { return this.containersAllocated; } + public int getReservedContainers() { + return this.containersReserved; + } + + public int getPendingContainers() { + return this.containersPending; + } + public long getTotalMB() { return this.totalMB; } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java index a6c1fcaac9..9d4d77ce08 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java @@ -32,6 +32,11 @@ public class UserMetricsInfo { protected int appsSubmitted; + protected int appsCompleted; + protected int appsPending; + protected int appsRunning; + protected int appsFailed; + protected int appsKilled; protected int runningContainers; protected int pendingContainers; protected int reservedContainers; @@ -54,10 +59,18 @@ public UserMetricsInfo(final ResourceManager rm, final RMContext rmContext, if (userMetrics != null) { this.userMetricsAvailable = true; + this.appsSubmitted = userMetrics.getAppsSubmitted(); + this.appsCompleted = metrics.getAppsCompleted(); + this.appsPending = metrics.getAppsPending(); + this.appsRunning = metrics.getAppsRunning(); + this.appsFailed = metrics.getAppsFailed(); + this.appsKilled = metrics.getAppsKilled(); + this.runningContainers = userMetrics.getAllocatedContainers(); this.pendingContainers = userMetrics.getPendingContainers(); this.reservedContainers = userMetrics.getReservedContainers(); + this.reservedMB = userMetrics.getReservedMB(); this.pendingMB = userMetrics.getPendingMB(); this.allocatedMB = userMetrics.getAllocatedMB(); @@ -72,6 +85,26 @@ public int getAppsSubmitted() { return this.appsSubmitted; } + public int getAppsCompleted() { + return appsCompleted; + } + + public int getAppsPending() { + return appsPending; + } + + public int getAppsRunning() { + return appsRunning; + } + + public int getAppsFailed() { + return appsFailed; + } + + public int getAppsKilled() { + return appsKilled; + } + public long getReservedMB() { return this.reservedMB; } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index 7426d2b273..57e0c69385 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -142,7 +142,7 @@ public void testLimitsComputation() throws Exception { CapacityScheduler.parseQueue(csContext, csConf, null, "root", queues, queues, CapacityScheduler.queueComparator, - CapacityScheduler.applicationComparator, + CapacityScheduler.applicationComparator, TestUtils.spyHook); LeafQueue queue = (LeafQueue)queues.get(A); @@ -163,6 +163,10 @@ public void testLimitsComputation() throws Exception { expectedMaxActiveApps * (queue.getUserLimit() / 100.0f) * queue.getUserLimitFactor()), queue.getMaximumActiveApplicationsPerUser()); + assertEquals( + (int)(clusterResource.getMemory() * queue.getAbsoluteCapacity()), + queue.getMetrics().getAvailableMB() + ); // Add some nodes to the cluster & test new limits clusterResource = Resources.createResource(120 * 16 * GB); @@ -178,6 +182,10 @@ public void testLimitsComputation() throws Exception { (int)Math.ceil(expectedMaxActiveApps * (queue.getUserLimit() / 100.0f) * queue.getUserLimitFactor()), queue.getMaximumActiveApplicationsPerUser()); + assertEquals( + (int)(clusterResource.getMemory() * queue.getAbsoluteCapacity()), + queue.getMetrics().getAvailableMB() + ); } @Test diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java index 5d97adae5d..4922419c2f 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java @@ -48,7 +48,7 @@ public class TestNodesPage { // Number of Actual Table Headers for NodesPage.NodesBlock might change in // future. In that case this value should be adjusted to the new value. - final int numberOfThInMetricsTable = 10; + final int numberOfThInMetricsTable = 13; final int numberOfActualTableHeaders = 10; private Injector injector; diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java index d09645a97d..fc73c0251a 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java @@ -361,6 +361,7 @@ public void verifyClusterMetricsXML(String xml) throws JSONException, verifyClusterMetrics( WebServicesTestUtils.getXmlInt(element, "appsSubmitted"), + WebServicesTestUtils.getXmlInt(element, "appsCompleted"), WebServicesTestUtils.getXmlInt(element, "reservedMB"), WebServicesTestUtils.getXmlInt(element, "availableMB"), WebServicesTestUtils.getXmlInt(element, "allocatedMB"), @@ -379,8 +380,9 @@ public void verifyClusterMetricsJSON(JSONObject json) throws JSONException, Exception { assertEquals("incorrect number of elements", 1, json.length()); JSONObject clusterinfo = json.getJSONObject("clusterMetrics"); - assertEquals("incorrect number of elements", 12, clusterinfo.length()); - verifyClusterMetrics(clusterinfo.getInt("appsSubmitted"), + assertEquals("incorrect number of elements", 19, clusterinfo.length()); + verifyClusterMetrics( + clusterinfo.getInt("appsSubmitted"), clusterinfo.getInt("appsCompleted"), clusterinfo.getInt("reservedMB"), clusterinfo.getInt("availableMB"), clusterinfo.getInt("allocatedMB"), clusterinfo.getInt("containersAllocated"), @@ -390,7 +392,8 @@ public void verifyClusterMetricsJSON(JSONObject json) throws JSONException, clusterinfo.getInt("rebootedNodes"),clusterinfo.getInt("activeNodes")); } - public void verifyClusterMetrics(int sub, int reservedMB, int availableMB, + public void verifyClusterMetrics(int submittedApps, int completedApps, + int reservedMB, int availableMB, int allocMB, int containersAlloc, int totalMB, int totalNodes, int lostNodes, int unhealthyNodes, int decommissionedNodes, int rebootedNodes, int activeNodes) throws JSONException, Exception { @@ -404,7 +407,9 @@ public void verifyClusterMetrics(int sub, int reservedMB, int availableMB, + metrics.getAllocatedMB(); assertEquals("appsSubmitted doesn't match", - metrics.getAppsSubmitted(), sub); + metrics.getAppsSubmitted(), submittedApps); + assertEquals("appsCompleted doesn't match", + metrics.getAppsCompleted(), completedApps); assertEquals("reservedMB doesn't match", metrics.getReservedMB(), reservedMB); assertEquals("availableMB doesn't match",