big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-254. Update fair scheduler web UI for hierarchical queues. (sandyr via tucu)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1423742 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Alejandro Abdelnur 2012-12-19 03:04:39 +00:00
parent 5032a694ed
commit 595d52cf37
5 changed files with 124 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -76,6 +76,9 @@ Release 2.0.3-alpha - Unreleased
YARN-129. Simplify classpath construction for mini YARN tests. (tomwhite)
YARN-254. Update fair scheduler web UI for hierarchical queues.
(sandyr via tucu)
OPTIMIZATIONS
BUG FIXES

89
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java
View File

@ -20,21 +20,21 @@
import static org.apache.hadoop.yarn.util.StringHelper.join;
import java.util.List;
import java.util.Collection;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerLeafQueueInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerQueueInfo;
import org.apache.hadoop.yarn.webapp.ResponseInfo;
import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.LI;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.UL;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
import org.apache.hadoop.yarn.webapp.view.HtmlPage.Page;
import org.apache.hadoop.yarn.webapp.view.HtmlPage._;
import com.google.inject.Inject;
import com.google.inject.servlet.RequestScoped;
@ -50,16 +50,15 @@ public class FairSchedulerPage extends RmView {
@RequestScoped
static class FSQInfo {
FairSchedulerInfo fsinfo;
FairSchedulerQueueInfo qinfo;
}
static class QueueInfoBlock extends HtmlBlock {
final FairSchedulerQueueInfo qinfo;
static class LeafQueueBlock extends HtmlBlock {
final FairSchedulerLeafQueueInfo qinfo;
@Inject QueueInfoBlock(ViewContext ctx, FSQInfo info) {
@Inject LeafQueueBlock(ViewContext ctx, FSQInfo info) {
super(ctx);
qinfo = (FairSchedulerQueueInfo) info.qinfo;
qinfo = (FairSchedulerLeafQueueInfo)info.qinfo;
}
@Override
@ -83,6 +82,47 @@ protected void render(Block html) {
}
}
static class QueueBlock extends HtmlBlock {
final FSQInfo fsqinfo;
@Inject QueueBlock(FSQInfo info) {
fsqinfo = info;
}
@Override
public void render(Block html) {
Collection<FairSchedulerQueueInfo> subQueues = fsqinfo.qinfo.getChildQueues();
UL<Hamlet> ul = html.ul("#pq");
for (FairSchedulerQueueInfo info : subQueues) {
float capacity = info.getMaxResourcesFraction();
float fairShare = info.getFairShareFraction();
float used = info.getUsedFraction();
LI<UL<Hamlet>> li = ul.
li().
a(_Q).$style(width(capacity * Q_MAX_WIDTH)).
$title(join("Fair Share:", percent(fairShare))).
span().$style(join(Q_GIVEN, ";font-size:1px;", width(fairShare/capacity))).
_('.')._().
span().$style(join(width(used/capacity),
";font-size:1px;left:0%;", used > fairShare ? Q_OVER : Q_UNDER)).
_('.')._().
span(".q", info.getQueueName())._().
span().$class("qstats").$style(left(Q_STATS_POS)).
_(join(percent(used), " used"))._();
fsqinfo.qinfo = info;
if (info instanceof FairSchedulerLeafQueueInfo) {
li.ul("#lq").li()._(LeafQueueBlock.class)._()._();
} else {
li._(QueueBlock.class);
}
li._();
}
ul._();
}
}
static class QueuesBlock extends HtmlBlock {
final FairScheduler fs;
final FSQInfo fsqinfo;
@ -92,7 +132,8 @@ static class QueuesBlock extends HtmlBlock {
fsqinfo = info;
}
@Override public void render(Block html) {
@Override
public void render(Block html) {
html._(MetricsOverviewTable.class);
UL<DIV<DIV<Hamlet>>> ul = html.
div("#cs-wrapper.ui-widget").
@ -108,8 +149,8 @@ static class QueuesBlock extends HtmlBlock {
span(".q", "default")._()._();
} else {
FairSchedulerInfo sinfo = new FairSchedulerInfo(fs);
fsqinfo.fsinfo = sinfo;
fsqinfo.qinfo = null;
fsqinfo.qinfo = sinfo.getRootQueueInfo();
float used = fsqinfo.qinfo.getUsedFraction();
ul.
li().$style("margin-bottom: 1em").
@ -122,29 +163,15 @@ static class QueuesBlock extends HtmlBlock {
_("Used (over fair share)")._().
span().$class("qlegend ui-corner-all ui-state-default").
_("Max Capacity")._().
_();
List<FairSchedulerQueueInfo> subQueues = fsqinfo.fsinfo.getQueueInfos();
for (FairSchedulerQueueInfo info : subQueues) {
fsqinfo.qinfo = info;
float capacity = info.getMaxResourcesFraction();
float fairShare = info.getFairShareFraction();
float used = info.getUsedFraction();
ul.
_().
li().
a(_Q).$style(width(capacity * Q_MAX_WIDTH)).
$title(join("Fair Share:", percent(fairShare))).
span().$style(join(Q_GIVEN, ";font-size:1px;", width(fairShare/capacity))).
_('.')._().
span().$style(join(width(used/capacity),
";font-size:1px;left:0%;", used > fairShare ? Q_OVER : Q_UNDER)).
_('.')._().
span(".q", info.getQueueName())._().
a(_Q).$style(width(Q_MAX_WIDTH)).
span().$style(join(width(used), ";left:0%;",
used > 1 ? Q_OVER : Q_UNDER))._(".")._().
span(".q", "root")._().
span().$class("qstats").$style(left(Q_STATS_POS)).
_(join(percent(used), " used"))._().
ul("#lq").li()._(QueueInfoBlock.class)._()._().
_();
}
_(QueueBlock.class)._();
}
ul._()._().
script().$type("text/javascript").

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java
View File

@ -18,33 +18,23 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
public class FairSchedulerInfo {
private List<FairSchedulerQueueInfo> queueInfos;
private FairScheduler scheduler;
public FairSchedulerInfo(FairScheduler fs) {
scheduler = fs;
Collection<FSLeafQueue> queues = fs.getQueueManager().getLeafQueues();
queueInfos = new ArrayList<FairSchedulerQueueInfo>();
for (FSLeafQueue queue : queues) {
queueInfos.add(new FairSchedulerQueueInfo(queue, fs));
}
}
public List<FairSchedulerQueueInfo> getQueueInfos() {
return queueInfos;
}
public int getAppFairShare(ApplicationAttemptId appAttemptId) {
return scheduler.getSchedulerApp(appAttemptId).
getAppSchedulable().getFairShare().getMemory();
}
public FairSchedulerQueueInfo getRootQueueInfo() {
return new FairSchedulerQueueInfo(scheduler.getQueueManager().
getRootQueue(), scheduler);
}
}

32
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java Normal file
View File

@ -0,0 +1,32 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import java.util.Collection;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AppSchedulable;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
public class FairSchedulerLeafQueueInfo extends FairSchedulerQueueInfo {
private int numPendingApps;
private int numActiveApps;
public FairSchedulerLeafQueueInfo(FSLeafQueue queue, FairScheduler scheduler) {
super(queue, scheduler);
Collection<AppSchedulable> apps = queue.getAppSchedulables();
for (AppSchedulable app : apps) {
if (app.getApp().isPending()) {
numPendingApps++;
} else {
numActiveApps++;
}
}
}
public int getNumActiveApplications() {
return numPendingApps;
}
public int getNumPendingApplications() {
return numActiveApps;
}
}

40
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
View File

@ -18,19 +18,18 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import java.util.ArrayList;
import java.util.Collection;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AppSchedulable;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager;
public class FairSchedulerQueueInfo {
private int numPendingApps;
private int numActiveApps;
private int fairShare;
private int minShare;
private int maxShare;
@ -48,16 +47,9 @@ public class FairSchedulerQueueInfo {
private String queueName;
public FairSchedulerQueueInfo(FSLeafQueue queue, FairScheduler scheduler) {
Collection<AppSchedulable> apps = queue.getAppSchedulables();
for (AppSchedulable app : apps) {
if (app.getApp().isPending()) {
numPendingApps++;
} else {
numActiveApps++;
}
}
private Collection<FairSchedulerQueueInfo> childInfos;
public FairSchedulerQueueInfo(FSQueue queue, FairScheduler scheduler) {
QueueManager manager = scheduler.getQueueManager();
queueName = queue.getName();
@ -81,6 +73,16 @@ public FairSchedulerQueueInfo(FSLeafQueue queue, FairScheduler scheduler) {
fractionMinShare = (float)minShare / clusterMaxMem;
maxApps = manager.getQueueMaxApps(queueName);
Collection<FSQueue> childQueues = queue.getChildQueues();
childInfos = new ArrayList<FairSchedulerQueueInfo>();
for (FSQueue child : childQueues) {
if (child instanceof FSLeafQueue) {
childInfos.add(new FairSchedulerLeafQueueInfo((FSLeafQueue)child, scheduler));
} else {
childInfos.add(new FairSchedulerQueueInfo(child, scheduler));
}
}
}
/**
@ -97,14 +99,6 @@ public int getFairShare() {
return fairShare;
}
public int getNumActiveApplications() {
return numPendingApps;
}
public int getNumPendingApplications() {
return numActiveApps;
}
public Resource getMinResources() {
return minResources;
}
@ -148,4 +142,8 @@ public float getUsedFraction() {
public float getMaxResourcesFraction() {
return (float)maxShare / clusterMaxMem;
}
public Collection<FairSchedulerQueueInfo> getChildQueues() {
return childInfos;
}
}