From 595d52cf3727ca3f8883d30e50f67975b1a3f796 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Wed, 19 Dec 2012 03:04:39 +0000 Subject: [PATCH] YARN-254. Update fair scheduler web UI for hierarchical queues. (sandyr via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1423742 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../webapp/FairSchedulerPage.java | 97 ++++++++++++------- .../webapp/dao/FairSchedulerInfo.java | 20 +--- .../dao/FairSchedulerLeafQueueInfo.java | 32 ++++++ .../webapp/dao/FairSchedulerQueueInfo.java | 46 +++++---- 5 files changed, 124 insertions(+), 74 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 892ea46510..6eb9379308 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -76,6 +76,9 @@ Release 2.0.3-alpha - Unreleased YARN-129. Simplify classpath construction for mini YARN tests. (tomwhite) + YARN-254. Update fair scheduler web UI for hierarchical queues. + (sandyr via tucu) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java index 8872fab097..b36fd9a4c2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java @@ -20,21 +20,21 @@ import static org.apache.hadoop.yarn.util.StringHelper.join; -import java.util.List; +import java.util.Collection; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerLeafQueueInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerQueueInfo; import org.apache.hadoop.yarn.webapp.ResponseInfo; import org.apache.hadoop.yarn.webapp.SubView; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.LI; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.UL; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; import org.apache.hadoop.yarn.webapp.view.InfoBlock; -import org.apache.hadoop.yarn.webapp.view.HtmlPage.Page; -import org.apache.hadoop.yarn.webapp.view.HtmlPage._; import com.google.inject.Inject; import com.google.inject.servlet.RequestScoped; @@ -50,16 +50,15 @@ public class FairSchedulerPage extends RmView { @RequestScoped static class FSQInfo { - FairSchedulerInfo fsinfo; FairSchedulerQueueInfo qinfo; } - static class QueueInfoBlock extends HtmlBlock { - final FairSchedulerQueueInfo qinfo; + static class LeafQueueBlock extends HtmlBlock { + final FairSchedulerLeafQueueInfo qinfo; - @Inject QueueInfoBlock(ViewContext ctx, FSQInfo info) { + @Inject LeafQueueBlock(ViewContext ctx, FSQInfo info) { super(ctx); - qinfo = (FairSchedulerQueueInfo) info.qinfo; + qinfo = (FairSchedulerLeafQueueInfo)info.qinfo; } @Override @@ -83,6 +82,47 @@ protected void render(Block html) { } } + static class QueueBlock extends HtmlBlock { + final FSQInfo fsqinfo; + + @Inject QueueBlock(FSQInfo info) { + fsqinfo = info; + } + + @Override + public void render(Block html) { + Collection subQueues = fsqinfo.qinfo.getChildQueues(); + UL ul = html.ul("#pq"); + for (FairSchedulerQueueInfo info : subQueues) { + float capacity = info.getMaxResourcesFraction(); + float fairShare = info.getFairShareFraction(); + float used = info.getUsedFraction(); + LI> li = ul. + li(). + a(_Q).$style(width(capacity * Q_MAX_WIDTH)). + $title(join("Fair Share:", percent(fairShare))). + span().$style(join(Q_GIVEN, ";font-size:1px;", width(fairShare/capacity))). + _('.')._(). + span().$style(join(width(used/capacity), + ";font-size:1px;left:0%;", used > fairShare ? Q_OVER : Q_UNDER)). + _('.')._(). + span(".q", info.getQueueName())._(). + span().$class("qstats").$style(left(Q_STATS_POS)). + _(join(percent(used), " used"))._(); + + fsqinfo.qinfo = info; + if (info instanceof FairSchedulerLeafQueueInfo) { + li.ul("#lq").li()._(LeafQueueBlock.class)._()._(); + } else { + li._(QueueBlock.class); + } + li._(); + } + + ul._(); + } + } + static class QueuesBlock extends HtmlBlock { final FairScheduler fs; final FSQInfo fsqinfo; @@ -91,8 +131,9 @@ static class QueuesBlock extends HtmlBlock { fs = (FairScheduler)rm.getResourceScheduler(); fsqinfo = info; } - - @Override public void render(Block html) { + + @Override + public void render(Block html) { html._(MetricsOverviewTable.class); UL>> ul = html. div("#cs-wrapper.ui-widget"). @@ -108,8 +149,8 @@ static class QueuesBlock extends HtmlBlock { span(".q", "default")._()._(); } else { FairSchedulerInfo sinfo = new FairSchedulerInfo(fs); - fsqinfo.fsinfo = sinfo; - fsqinfo.qinfo = null; + fsqinfo.qinfo = sinfo.getRootQueueInfo(); + float used = fsqinfo.qinfo.getUsedFraction(); ul. li().$style("margin-bottom: 1em"). @@ -122,29 +163,15 @@ static class QueuesBlock extends HtmlBlock { _("Used (over fair share)")._(). span().$class("qlegend ui-corner-all ui-state-default"). _("Max Capacity")._(). - _(); - - List subQueues = fsqinfo.fsinfo.getQueueInfos(); - for (FairSchedulerQueueInfo info : subQueues) { - fsqinfo.qinfo = info; - float capacity = info.getMaxResourcesFraction(); - float fairShare = info.getFairShareFraction(); - float used = info.getUsedFraction(); - ul. - li(). - a(_Q).$style(width(capacity * Q_MAX_WIDTH)). - $title(join("Fair Share:", percent(fairShare))). - span().$style(join(Q_GIVEN, ";font-size:1px;", width(fairShare/capacity))). - _('.')._(). - span().$style(join(width(used/capacity), - ";font-size:1px;left:0%;", used > fairShare ? Q_OVER : Q_UNDER)). - _('.')._(). - span(".q", info.getQueueName())._(). - span().$class("qstats").$style(left(Q_STATS_POS)). - _(join(percent(used), " used"))._(). - ul("#lq").li()._(QueueInfoBlock.class)._()._(). - _(); - } + _(). + li(). + a(_Q).$style(width(Q_MAX_WIDTH)). + span().$style(join(width(used), ";left:0%;", + used > 1 ? Q_OVER : Q_UNDER))._(".")._(). + span(".q", "root")._(). + span().$class("qstats").$style(left(Q_STATS_POS)). + _(join(percent(used), " used"))._(). + _(QueueBlock.class)._(); } ul._()._(). script().$type("text/javascript"). diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java index 4fe19ca13d..e1fac4a2cf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java @@ -18,33 +18,23 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; public class FairSchedulerInfo { - private List queueInfos; private FairScheduler scheduler; public FairSchedulerInfo(FairScheduler fs) { scheduler = fs; - Collection queues = fs.getQueueManager().getLeafQueues(); - queueInfos = new ArrayList(); - for (FSLeafQueue queue : queues) { - queueInfos.add(new FairSchedulerQueueInfo(queue, fs)); - } - } - - public List getQueueInfos() { - return queueInfos; } public int getAppFairShare(ApplicationAttemptId appAttemptId) { return scheduler.getSchedulerApp(appAttemptId). getAppSchedulable().getFairShare().getMemory(); } + + public FairSchedulerQueueInfo getRootQueueInfo() { + return new FairSchedulerQueueInfo(scheduler.getQueueManager(). + getRootQueue(), scheduler); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java new file mode 100644 index 0000000000..bee1cfd986 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java @@ -0,0 +1,32 @@ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.Collection; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AppSchedulable; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue; + +public class FairSchedulerLeafQueueInfo extends FairSchedulerQueueInfo { + private int numPendingApps; + private int numActiveApps; + + public FairSchedulerLeafQueueInfo(FSLeafQueue queue, FairScheduler scheduler) { + super(queue, scheduler); + Collection apps = queue.getAppSchedulables(); + for (AppSchedulable app : apps) { + if (app.getApp().isPending()) { + numPendingApps++; + } else { + numActiveApps++; + } + } + } + + public int getNumActiveApplications() { + return numPendingApps; + } + + public int getNumPendingApplications() { + return numActiveApps; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java index 35749427d0..3cab1da33c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java @@ -18,19 +18,18 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.ArrayList; import java.util.Collection; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AppSchedulable; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager; -public class FairSchedulerQueueInfo { - private int numPendingApps; - private int numActiveApps; - +public class FairSchedulerQueueInfo { private int fairShare; private int minShare; private int maxShare; @@ -48,16 +47,9 @@ public class FairSchedulerQueueInfo { private String queueName; - public FairSchedulerQueueInfo(FSLeafQueue queue, FairScheduler scheduler) { - Collection apps = queue.getAppSchedulables(); - for (AppSchedulable app : apps) { - if (app.getApp().isPending()) { - numPendingApps++; - } else { - numActiveApps++; - } - } - + private Collection childInfos; + + public FairSchedulerQueueInfo(FSQueue queue, FairScheduler scheduler) { QueueManager manager = scheduler.getQueueManager(); queueName = queue.getName(); @@ -81,6 +73,16 @@ public FairSchedulerQueueInfo(FSLeafQueue queue, FairScheduler scheduler) { fractionMinShare = (float)minShare / clusterMaxMem; maxApps = manager.getQueueMaxApps(queueName); + + Collection childQueues = queue.getChildQueues(); + childInfos = new ArrayList(); + for (FSQueue child : childQueues) { + if (child instanceof FSLeafQueue) { + childInfos.add(new FairSchedulerLeafQueueInfo((FSLeafQueue)child, scheduler)); + } else { + childInfos.add(new FairSchedulerQueueInfo(child, scheduler)); + } + } } /** @@ -96,15 +98,7 @@ public float getFairShareFraction() { public int getFairShare() { return fairShare; } - - public int getNumActiveApplications() { - return numPendingApps; - } - - public int getNumPendingApplications() { - return numActiveApps; - } - + public Resource getMinResources() { return minResources; } @@ -148,4 +142,8 @@ public float getUsedFraction() { public float getMaxResourcesFraction() { return (float)maxShare / clusterMaxMem; } + + public Collection getChildQueues() { + return childInfos; + } }