big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-3747. Initialize queue metrics upfront and added start/finish time to RM Web-UI.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1240886 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Arun Murthy 2012-02-06 03:38:44 +00:00
parent f88574acde
commit c7007a8a35
13 changed files with 260 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -705,6 +705,9 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3765. FifoScheduler does not respect yarn.scheduler.fifo.minimum-
allocation-mb setting (Hitesh Shah via mahadev)
MAPREDUCE-3747. Initialize queue metrics upfront and added start/finish
time to RM Web-UI. (acmurthy)
Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES

12
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
View File

@ -98,6 +98,12 @@ public interface CSQueue
*/
public float getUsedCapacity();
/**
* Set used capacity of the queue.
* @param usedCapacity used capacity of the queue
*/
public void setUsedCapacity(float usedCapacity);
/**
* Get the currently utilized resources in the cluster
* by the queue and children (if any).
@ -114,6 +120,12 @@ public interface CSQueue
*/
public float getUtilization();
/**
* Get the current <em>utilization</em> of the queue.
* @param utilization queue utilization
*/
public void setUtilization(float utilization);
/**
* Get the current run-state of the queue
* @return current run-state

38
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
View File

@ -17,7 +17,9 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import org.apache.hadoop.yarn.Lock;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
class CSQueueUtils {
@ -65,4 +67,40 @@ public static int computeMaxActiveApplicationsPerUser(
1);
}
@Lock(CSQueue.class)
public static void updateQueueStatistics(
final CSQueue childQueue, final CSQueue parentQueue,
final Resource clusterResource, final Resource minimumAllocation) {
final int clusterMemory = clusterResource.getMemory();
final int usedMemory = childQueue.getUsedResources().getMemory();
float queueLimit = 0.0f;
float utilization = 0.0f;
float usedCapacity = 0.0f;
if (clusterMemory > 0) {
queueLimit = clusterMemory * childQueue.getAbsoluteCapacity();
final float parentAbsoluteCapacity =
(parentQueue == null) ? 1.0f : parentQueue.getAbsoluteCapacity();
utilization = (usedMemory / queueLimit);
usedCapacity = (usedMemory / (clusterMemory * parentAbsoluteCapacity));
}
childQueue.setUtilization(utilization);
childQueue.setUsedCapacity(usedCapacity);
int available =
Math.max((roundUp(minimumAllocation, (int)queueLimit) - usedMemory), 0);
childQueue.getMetrics().setAvailableResourcesToQueue(
Resources.createResource(available));
}
public static int roundUp(Resource minimumAllocation, int memory) {
int minMemory = minimumAllocation.getMemory();
return LeafQueue.divideAndCeil(memory, minMemory) * minMemory;
}
public static int roundDown(Resource minimumAllocation, int memory) {
int minMemory = minimumAllocation.getMemory();
return (memory / minMemory) * minMemory;
}
}

66
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
View File

@ -180,7 +180,9 @@ public LeafQueue(CapacitySchedulerContext cs,
Map<QueueACL, AccessControlList> acls =
cs.getConfiguration().getAcls(getQueuePath());
setupQueueConfigs(capacity, absoluteCapacity,
setupQueueConfigs(
cs.getClusterResources(),
capacity, absoluteCapacity,
maximumCapacity, absoluteMaxCapacity,
userLimit, userLimitFactor,
maxApplications, maxApplicationsPerUser,
@ -198,6 +200,7 @@ public LeafQueue(CapacitySchedulerContext cs,
}
private synchronized void setupQueueConfigs(
Resource clusterResource,
float capacity, float absoluteCapacity,
float maximumCapacity, float absoluteMaxCapacity,
int userLimit, float userLimitFactor,
@ -235,6 +238,10 @@ private synchronized void setupQueueConfigs(
for (Map.Entry<QueueACL, AccessControlList> e : acls.entrySet()) {
aclsString.append(e.getKey() + ":" + e.getValue().getAclString());
}
// Update metrics
CSQueueUtils.updateQueueStatistics(
this, parent, clusterResource, minimumAllocation);
LOG.info("Initializing " + queueName + "\n" +
"capacity = " + capacity +
@ -386,11 +393,11 @@ public List<CSQueue> getChildQueues() {
return null;
}
synchronized void setUtilization(float utilization) {
public synchronized void setUtilization(float utilization) {
this.utilization = utilization;
}
synchronized void setUsedCapacity(float usedCapacity) {
public synchronized void setUsedCapacity(float usedCapacity) {
this.usedCapacity = usedCapacity;
}
@ -534,7 +541,9 @@ public synchronized void reinitialize(CSQueue queue, Resource clusterResource)
}
LeafQueue leafQueue = (LeafQueue)queue;
setupQueueConfigs(leafQueue.capacity, leafQueue.absoluteCapacity,
setupQueueConfigs(
clusterResource,
leafQueue.capacity, leafQueue.absoluteCapacity,
leafQueue.maximumCapacity, leafQueue.absoluteMaxCapacity,
leafQueue.userLimit, leafQueue.userLimitFactor,
leafQueue.maxApplications,
@ -542,8 +551,6 @@ public synchronized void reinitialize(CSQueue queue, Resource clusterResource)
leafQueue.getMaximumActiveApplications(),
leafQueue.getMaximumActiveApplicationsPerUser(),
leafQueue.state, leafQueue.acls);
updateResource(clusterResource);
}
@Override
@ -883,7 +890,8 @@ private Resource computeUserLimitAndSetHeadroom(
Resource queueMaxCap = // Queue Max-Capacity
Resources.createResource(
roundDown((int)(absoluteMaxCapacity * clusterResource.getMemory()))
CSQueueUtils.roundDown(minimumAllocation,
(int)(absoluteMaxCapacity * clusterResource.getMemory()))
);
Resource userConsumed = getUser(user).getConsumedResources();
@ -904,16 +912,6 @@ private Resource computeUserLimitAndSetHeadroom(
return userLimit;
}
private int roundUp(int memory) {
int minMemory = minimumAllocation.getMemory();
return divideAndCeil(memory, minMemory) * minMemory;
}
private int roundDown(int memory) {
int minMemory = minimumAllocation.getMemory();
return (memory / minMemory) * minMemory;
}
@Lock(NoLock.class)
private Resource computeUserLimit(SchedulerApp application,
Resource clusterResource, Resource required) {
@ -927,8 +925,11 @@ private Resource computeUserLimit(SchedulerApp application,
// Allow progress for queues with miniscule capacity
final int queueCapacity =
Math.max(
roundUp((int)(absoluteCapacity * clusterResource.getMemory())),
required.getMemory());
CSQueueUtils.roundUp(
minimumAllocation,
(int)(absoluteCapacity * clusterResource.getMemory())),
required.getMemory()
);
final int consumed = usedResources.getMemory();
final int currentCapacity =
@ -943,7 +944,8 @@ private Resource computeUserLimit(SchedulerApp application,
final int activeUsers = activeUsersManager.getNumActiveUsers();
int limit =
roundUp(
CSQueueUtils.roundUp(
minimumAllocation,
Math.min(
Math.max(divideAndCeil(currentCapacity, activeUsers),
divideAndCeil((int)userLimit*currentCapacity, 100)),
@ -991,7 +993,7 @@ private synchronized boolean assignToUser(String userName, Resource limit) {
return true;
}
private static int divideAndCeil(int a, int b) {
static int divideAndCeil(int a, int b) {
if (b == 0) {
LOG.info("divideAndCeil called with a=" + a + " b=" + b);
return 0;
@ -1325,7 +1327,8 @@ synchronized void allocateResource(Resource clusterResource,
SchedulerApp application, Resource resource) {
// Update queue metrics
Resources.addTo(usedResources, resource);
updateResource(clusterResource);
CSQueueUtils.updateQueueStatistics(
this, parent, clusterResource, minimumAllocation);
++numContainers;
// Update user metrics
@ -1349,7 +1352,8 @@ synchronized void releaseResource(Resource clusterResource,
SchedulerApp application, Resource resource) {
// Update queue metrics
Resources.subtractFrom(usedResources, resource);
updateResource(clusterResource);
CSQueueUtils.updateQueueStatistics(
this, parent, clusterResource, minimumAllocation);
--numContainers;
// Update user metrics
@ -1374,6 +1378,10 @@ public synchronized void updateClusterResource(Resource clusterResource) {
CSQueueUtils.computeMaxActiveApplicationsPerUser(
maxActiveApplications, userLimit, userLimitFactor);
// Update metrics
CSQueueUtils.updateQueueStatistics(
this, parent, clusterResource, minimumAllocation);
// Update application properties
for (SchedulerApp application : activeApplications) {
synchronized (application) {
@ -1383,18 +1391,6 @@ public synchronized void updateClusterResource(Resource clusterResource) {
}
}
private synchronized void updateResource(Resource clusterResource) {
float queueLimit = clusterResource.getMemory() * absoluteCapacity;
setUtilization(usedResources.getMemory() / queueLimit);
setUsedCapacity(usedResources.getMemory()
/ (clusterResource.getMemory() * parent.getAbsoluteCapacity()));
Resource resourceLimit =
Resources.createResource(roundUp((int)queueLimit));
metrics.setAvailableResourcesToQueue(
Resources.subtractFrom(resourceLimit, usedResources));
}
@Override
public QueueMetrics getMetrics() {
return metrics;

51
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
View File

@ -97,7 +97,8 @@ public class ParentQueue implements CSQueue {
RecordFactoryProvider.getRecordFactory(null);
public ParentQueue(CapacitySchedulerContext cs,
String queueName, Comparator<CSQueue> comparator, CSQueue parent, CSQueue old) {
String queueName, Comparator<CSQueue> comparator,
CSQueue parent, CSQueue old) {
minimumAllocation = cs.getMinimumResourceCapability();
this.parent = parent;
@ -137,7 +138,8 @@ public ParentQueue(CapacitySchedulerContext cs,
this.queueInfo.setQueueName(queueName);
this.queueInfo.setChildQueues(new ArrayList<QueueInfo>());
setupQueueConfigs(capacity, absoluteCapacity,
setupQueueConfigs(cs.getClusterResources(),
capacity, absoluteCapacity,
maximumCapacity, absoluteMaxCapacity, state, acls);
this.queueComparator = comparator;
@ -149,9 +151,10 @@ public ParentQueue(CapacitySchedulerContext cs,
}
private synchronized void setupQueueConfigs(
float capacity, float absoluteCapacity,
float maximumCapacity, float absoluteMaxCapacity,
QueueState state, Map<QueueACL, AccessControlList> acls
Resource clusterResource,
float capacity, float absoluteCapacity,
float maximumCapacity, float absoluteMaxCapacity,
QueueState state, Map<QueueACL, AccessControlList> acls
) {
// Sanity check
CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity);
@ -174,6 +177,10 @@ private synchronized void setupQueueConfigs(
aclsString.append(e.getKey() + ":" + e.getValue().getAclString());
}
// Update metrics
CSQueueUtils.updateQueueStatistics(
this, parent, clusterResource, minimumAllocation);
LOG.info(queueName +
", capacity=" + capacity +
", asboluteCapacity=" + absoluteCapacity +
@ -384,12 +391,10 @@ public synchronized void reinitialize(CSQueue queue, Resource clusterResource)
childQueues.addAll(currentChildQueues.values());
// Set new configs
setupQueueConfigs(parentQueue.capacity, parentQueue.absoluteCapacity,
setupQueueConfigs(clusterResource,
parentQueue.capacity, parentQueue.absoluteCapacity,
parentQueue.maximumCapacity, parentQueue.absoluteMaxCapacity,
parentQueue.state, parentQueue.acls);
// Update
updateResource(clusterResource);
}
Map<String, CSQueue> getQueues(Set<CSQueue> queues) {
@ -485,11 +490,11 @@ public synchronized void removeApplication(SchedulerApp application,
" #applications: " + getNumApplications());
}
synchronized void setUsedCapacity(float usedCapacity) {
public synchronized void setUsedCapacity(float usedCapacity) {
this.usedCapacity = usedCapacity;
}
synchronized void setUtilization(float utilization) {
public synchronized void setUtilization(float utilization) {
this.utilization = utilization;
}
@ -674,14 +679,16 @@ public void completedContainer(Resource clusterResource,
synchronized void allocateResource(Resource clusterResource,
Resource resource) {
Resources.addTo(usedResources, resource);
updateResource(clusterResource);
CSQueueUtils.updateQueueStatistics(
this, parent, clusterResource, minimumAllocation);
++numContainers;
}
synchronized void releaseResource(Resource clusterResource,
Resource resource) {
Resources.subtractFrom(usedResources, resource);
updateResource(clusterResource);
CSQueueUtils.updateQueueStatistics(
this, parent, clusterResource, minimumAllocation);
--numContainers;
}
@ -691,22 +698,12 @@ public synchronized void updateClusterResource(Resource clusterResource) {
for (CSQueue childQueue : childQueues) {
childQueue.updateClusterResource(clusterResource);
}
// Update metrics
CSQueueUtils.updateQueueStatistics(
this, parent, clusterResource, minimumAllocation);
}
private synchronized void updateResource(Resource clusterResource) {
float queueLimit = clusterResource.getMemory() * absoluteCapacity;
float parentAbsoluteCapacity =
(rootQueue) ? 1.0f : parent.getAbsoluteCapacity();
setUtilization(usedResources.getMemory() / queueLimit);
setUsedCapacity(usedResources.getMemory()
/ (clusterResource.getMemory() * parentAbsoluteCapacity));
Resource resourceLimit =
Resources.createResource((int)queueLimit);
metrics.setAvailableResourcesToQueue(
Resources.subtractFrom(resourceLimit, usedResources));
}
@Override
public QueueMetrics getMetrics() {
return metrics;

16
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java
View File

@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import static org.apache.hadoop.yarn.util.StringHelper.join;
import static org.apache.hadoop.yarn.util.StringHelper.sjoin;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI._PROGRESSBAR;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI._PROGRESSBAR_VALUE;
@ -27,6 +26,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
@ -52,11 +52,12 @@ class AppsBlock extends HtmlBlock {
th(".user", "User").
th(".name", "Name").
th(".queue", "Queue").
th(".starttime", "StartTime").
th(".finishtime", "FinishTime").
th(".state", "State").
th(".finalstatus", "FinalStatus").
th(".progress", "Progress").
th(".ui", "Tracking UI").
th(".note", "Note")._()._().
th(".ui", "Tracking UI")._()._().
tbody();
int i = 0;
String reqState = $(APP_STATE);
@ -67,6 +68,8 @@ class AppsBlock extends HtmlBlock {
}
AppInfo appInfo = new AppInfo(app, true);
String percent = String.format("%.1f", appInfo.getProgress());
String startTime = Times.format(appInfo.getStartTime());
String finishTime = Times.format(appInfo.getFinishTime());
tbody.
tr().
td().
@ -75,6 +78,10 @@ class AppsBlock extends HtmlBlock {
td(appInfo.getUser()).
td(appInfo.getName()).
td(appInfo.getQueue()).
td().
br().$title(startTime)._()._(startTime)._().
td().
br().$title(startTime)._()._(finishTime)._().
td(appInfo.getState()).
td(appInfo.getFinalStatus()).
td().
@ -85,8 +92,7 @@ class AppsBlock extends HtmlBlock {
$style(join("width:", percent, '%'))._()._()._().
td().
a(!appInfo.isTrackingUrlReady()?
"#" : appInfo.getTrackingUrlPretty(), appInfo.getTrackingUI())._().
td(appInfo.getNote())._();
"#" : appInfo.getTrackingUrlPretty(), appInfo.getTrackingUI())._()._();
if (list.rendering != Render.HTML && ++i >= 20) break;
}
tbody._()._();

49
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java
View File

@ -55,15 +55,19 @@ protected void render(Block html) {
//CSS in the correct spot
html.style(".metrics {margin-bottom:5px}");
ClusterMetricsInfo clusterMetrics = new ClusterMetricsInfo(this.rm, this.rmContext);
ClusterMetricsInfo clusterMetrics =
new ClusterMetricsInfo(this.rm, this.rmContext);
DIV<Hamlet> div = html.div().$class("metrics");
div.table("#metricsoverview").
div.h3("Cluster Metrics").
table("#metricsoverview").
thead().$class("ui-widget-header").
tr().
th().$class("ui-state-default")._("Apps Submitted")._().
th().$class("ui-state-default")._("Apps Pending")._().
th().$class("ui-state-default")._("Apps Running")._().
th().$class("ui-state-default")._("Apps Completed")._().
th().$class("ui-state-default")._("Containers Running")._().
th().$class("ui-state-default")._("Memory Used")._().
th().$class("ui-state-default")._("Memory Total")._().
@ -78,6 +82,14 @@ protected void render(Block html) {
tbody().$class("ui-widget-content").
tr().
td(String.valueOf(clusterMetrics.getAppsSubmitted())).
td(String.valueOf(clusterMetrics.getAppsPending())).
td(String.valueOf(clusterMetrics.getAppsRunning())).
td(
String.valueOf(
clusterMetrics.getAppsCompleted() +
clusterMetrics.getAppsFailed() + clusterMetrics.getAppsKilled()
)
).
td(String.valueOf(clusterMetrics.getContainersAllocated())).
td(StringUtils.byteDesc(clusterMetrics.getAllocatedMB() * BYTES_IN_MB)).
td(StringUtils.byteDesc(clusterMetrics.getTotalMB() * BYTES_IN_MB)).
@ -89,26 +101,38 @@ protected void render(Block html) {
td().a(url("nodes/rebooted"),String.valueOf(clusterMetrics.getRebootedNodes()))._().
_().
_()._();
String user = request().getRemoteUser();
if (user != null) {
UserMetricsInfo userMetrics = new UserMetricsInfo(this.rm, this.rmContext, user);
if (userMetrics.metricsAvailable()) {
div.table("#usermetricsoverview").
div.h3("User Metrics for " + user).
table("#usermetricsoverview").
thead().$class("ui-widget-header").
tr().
th().$class("ui-state-default")._("Apps Submitted ("+user+")")._().
th().$class("ui-state-default")._("Containers Running ("+user+")")._().
th().$class("ui-state-default")._("Containers Pending ("+user+")")._().
th().$class("ui-state-default")._("Containers Reserved ("+user+")")._().
th().$class("ui-state-default")._("Memory Used ("+user+")")._().
th().$class("ui-state-default")._("Memory Pending ("+user+")")._().
th().$class("ui-state-default")._("Memory Reserved ("+user+")")._().
th().$class("ui-state-default")._("Apps Submitted")._().
th().$class("ui-state-default")._("Apps Pending")._().
th().$class("ui-state-default")._("Apps Running")._().
th().$class("ui-state-default")._("Apps Completed")._().
th().$class("ui-state-default")._("Containers Running")._().
th().$class("ui-state-default")._("Containers Pending")._().
th().$class("ui-state-default")._("Containers Reserved")._().
th().$class("ui-state-default")._("Memory Used")._().
th().$class("ui-state-default")._("Memory Pending")._().
th().$class("ui-state-default")._("Memory Reserved")._().
_().
_().
tbody().$class("ui-widget-content").
tr().
td(String.valueOf(userMetrics.getAppsSubmitted())).
td(String.valueOf(userMetrics.getAppsPending())).
td(String.valueOf(userMetrics.getAppsRunning())).
td(
String.valueOf(
(userMetrics.getAppsCompleted() +
userMetrics.getAppsFailed() + userMetrics.getAppsKilled())
)
).
td(String.valueOf(userMetrics.getRunningContainers())).
td(String.valueOf(userMetrics.getPendingContainers())).
td(String.valueOf(userMetrics.getReservedContainers())).
@ -117,6 +141,7 @@ protected void render(Block html) {
td(StringUtils.byteDesc(userMetrics.getReservedMB() * BYTES_IN_MB)).
_().
_()._();
}
}

7
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java
View File

@ -63,10 +63,11 @@ protected Class<? extends SubView> content() {
private String appsTableInit() {
AppsList list = getInstance(AppsList.class);
// id, user, name, queue, state, progress, ui, note
// id, user, name, queue, starttime, finishtime, state, progress, ui
StringBuilder init = tableInit().
append(", aoColumns:[{sType:'title-numeric'}, null, null, null, null,").
append("null,{sType:'title-numeric', bSearchable:false}, null, null]");
append(", aoColumns:[{sType:'title-numeric'}, null, null, null, ").
append("null, null , null, ").
append("null,{sType:'title-numeric', bSearchable:false}, null]");
// Sort by id upon page load
init.append(", aaSorting: [[0, 'asc']]");

48
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
View File

@ -32,10 +32,20 @@
public class ClusterMetricsInfo {
protected int appsSubmitted;
protected int appsCompleted;
protected int appsPending;
protected int appsRunning;
protected int appsFailed;
protected int appsKilled;
protected long reservedMB;
protected long availableMB;
protected long allocatedMB;
protected int containersAllocated;
protected int containersReserved;
protected int containersPending;
protected long totalMB;
protected int totalNodes;
protected int lostNodes;
@ -53,10 +63,20 @@ public ClusterMetricsInfo(final ResourceManager rm, final RMContext rmContext) {
ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();
this.appsSubmitted = metrics.getAppsSubmitted();
this.appsCompleted = metrics.getAppsCompleted();
this.appsPending = metrics.getAppsPending();
this.appsRunning = metrics.getAppsRunning();
this.appsFailed = metrics.getAppsFailed();
this.appsKilled = metrics.getAppsKilled();
this.reservedMB = metrics.getReservedMB();
this.availableMB = metrics.getAvailableMB();
this.allocatedMB = metrics.getAllocatedMB();
this.containersAllocated = metrics.getAllocatedContainers();
this.containersPending = metrics.getPendingContainers();
this.containersReserved = metrics.getReservedContainers();
this.totalMB = availableMB + reservedMB + allocatedMB;
this.activeNodes = clusterMetrics.getNumActiveNMs();
this.lostNodes = clusterMetrics.getNumLostNMs();
@ -71,6 +91,26 @@ public int getAppsSubmitted() {
return this.appsSubmitted;
}
public int getAppsCompleted() {
return appsCompleted;
}
public int getAppsPending() {
return appsPending;
}
public int getAppsRunning() {
return appsRunning;
}
public int getAppsFailed() {
return appsFailed;
}
public int getAppsKilled() {
return appsKilled;
}
public long getReservedMB() {
return this.reservedMB;
}
@ -87,6 +127,14 @@ public int getContainersAllocated() {
return this.containersAllocated;
}
public int getReservedContainers() {
return this.containersReserved;
}
public int getPendingContainers() {
return this.containersPending;
}
public long getTotalMB() {
return this.totalMB;
}

33
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java
View File

@ -32,6 +32,11 @@
public class UserMetricsInfo {
protected int appsSubmitted;
protected int appsCompleted;
protected int appsPending;
protected int appsRunning;
protected int appsFailed;
protected int appsKilled;
protected int runningContainers;
protected int pendingContainers;
protected int reservedContainers;
@ -54,10 +59,18 @@ public UserMetricsInfo(final ResourceManager rm, final RMContext rmContext,
if (userMetrics != null) {
this.userMetricsAvailable = true;
this.appsSubmitted = userMetrics.getAppsSubmitted();
this.appsCompleted = metrics.getAppsCompleted();
this.appsPending = metrics.getAppsPending();
this.appsRunning = metrics.getAppsRunning();
this.appsFailed = metrics.getAppsFailed();
this.appsKilled = metrics.getAppsKilled();
this.runningContainers = userMetrics.getAllocatedContainers();
this.pendingContainers = userMetrics.getPendingContainers();
this.reservedContainers = userMetrics.getReservedContainers();
this.reservedMB = userMetrics.getReservedMB();
this.pendingMB = userMetrics.getPendingMB();
this.allocatedMB = userMetrics.getAllocatedMB();
@ -72,6 +85,26 @@ public int getAppsSubmitted() {
return this.appsSubmitted;
}
public int getAppsCompleted() {
return appsCompleted;
}
public int getAppsPending() {
return appsPending;
}
public int getAppsRunning() {
return appsRunning;
}
public int getAppsFailed() {
return appsFailed;
}
public int getAppsKilled() {
return appsKilled;
}
public long getReservedMB() {
return this.reservedMB;
}

10
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
View File

@ -142,7 +142,7 @@ public void testLimitsComputation() throws Exception {
CapacityScheduler.parseQueue(csContext, csConf, null, "root",
queues, queues,
CapacityScheduler.queueComparator,
CapacityScheduler.applicationComparator,
CapacityScheduler.applicationComparator,
TestUtils.spyHook);
LeafQueue queue = (LeafQueue)queues.get(A);
@ -163,6 +163,10 @@ public void testLimitsComputation() throws Exception {
expectedMaxActiveApps * (queue.getUserLimit() / 100.0f) *
queue.getUserLimitFactor()),
queue.getMaximumActiveApplicationsPerUser());
assertEquals(
(int)(clusterResource.getMemory() * queue.getAbsoluteCapacity()),
queue.getMetrics().getAvailableMB()
);
// Add some nodes to the cluster & test new limits
clusterResource = Resources.createResource(120 * 16 * GB);
@ -178,6 +182,10 @@ public void testLimitsComputation() throws Exception {
(int)Math.ceil(expectedMaxActiveApps *
(queue.getUserLimit() / 100.0f) * queue.getUserLimitFactor()),
queue.getMaximumActiveApplicationsPerUser());
assertEquals(
(int)(clusterResource.getMemory() * queue.getAbsoluteCapacity()),
queue.getMetrics().getAvailableMB()
);
}
@Test

2
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
View File

@ -48,7 +48,7 @@ public class TestNodesPage {
// Number of Actual Table Headers for NodesPage.NodesBlock might change in
// future. In that case this value should be adjusted to the new value.
final int numberOfThInMetricsTable = 10;
final int numberOfThInMetricsTable = 13;
final int numberOfActualTableHeaders = 10;
private Injector injector;

13
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
View File

@ -361,6 +361,7 @@ public void verifyClusterMetricsXML(String xml) throws JSONException,
verifyClusterMetrics(
WebServicesTestUtils.getXmlInt(element, "appsSubmitted"),
WebServicesTestUtils.getXmlInt(element, "appsCompleted"),
WebServicesTestUtils.getXmlInt(element, "reservedMB"),
WebServicesTestUtils.getXmlInt(element, "availableMB"),
WebServicesTestUtils.getXmlInt(element, "allocatedMB"),
@ -379,8 +380,9 @@ public void verifyClusterMetricsJSON(JSONObject json) throws JSONException,
Exception {
assertEquals("incorrect number of elements", 1, json.length());
JSONObject clusterinfo = json.getJSONObject("clusterMetrics");
assertEquals("incorrect number of elements", 12, clusterinfo.length());
verifyClusterMetrics(clusterinfo.getInt("appsSubmitted"),
assertEquals("incorrect number of elements", 19, clusterinfo.length());
verifyClusterMetrics(
clusterinfo.getInt("appsSubmitted"), clusterinfo.getInt("appsCompleted"),
clusterinfo.getInt("reservedMB"), clusterinfo.getInt("availableMB"),
clusterinfo.getInt("allocatedMB"),
clusterinfo.getInt("containersAllocated"),
@ -390,7 +392,8 @@ public void verifyClusterMetricsJSON(JSONObject json) throws JSONException,
clusterinfo.getInt("rebootedNodes"),clusterinfo.getInt("activeNodes"));
}
public void verifyClusterMetrics(int sub, int reservedMB, int availableMB,
public void verifyClusterMetrics(int submittedApps, int completedApps,
int reservedMB, int availableMB,
int allocMB, int containersAlloc, int totalMB, int totalNodes,
int lostNodes, int unhealthyNodes, int decommissionedNodes,
int rebootedNodes, int activeNodes) throws JSONException, Exception {
@ -404,7 +407,9 @@ public void verifyClusterMetrics(int sub, int reservedMB, int availableMB,
+ metrics.getAllocatedMB();
assertEquals("appsSubmitted doesn't match",
metrics.getAppsSubmitted(), sub);
metrics.getAppsSubmitted(), submittedApps);
assertEquals("appsCompleted doesn't match",
metrics.getAppsCompleted(), completedApps);
assertEquals("reservedMB doesn't match",
metrics.getReservedMB(), reservedMB);
assertEquals("availableMB doesn't match",