From 035937bee8ef57bc24b08068e7375ac45413a349 Mon Sep 17 00:00:00 2001 From: Arun Murthy Date: Fri, 9 Sep 2011 02:04:55 +0000 Subject: [PATCH] MAPREDUCE-2690. Web-page for FifoScheduler. Contributed by Eric Payne. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1166958 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 2 + .../scheduler/fifo/FifoScheduler.java | 2 + .../webapp/DefaultSchedulerPage.java | 167 +++++++++++++++++- .../resourcemanager/webapp/TestRMWebApp.java | 30 ++++ 4 files changed, 196 insertions(+), 5 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 73ab5bc84c..c2985d8799 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -255,6 +255,8 @@ Release 0.23.0 - Unreleased MAPREDUCE-2864. Normalize configuration variable names for YARN. (Robert Evans via acmurthy) + MAPREDUCE-2690. Web-page for FifoScheduler. (Eric Payne via acmurthy) + OPTIMIZATIONS MAPREDUCE-2026. Make JobTracker.getJobCounters() and diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index aba3020cf5..81f23633af 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -82,6 +82,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager; import org.apache.hadoop.yarn.util.BuilderUtils; +import org.apache.hadoop.yarn.api.records.QueueState; @LimitedPrivate("yarn") @Evolving @@ -144,6 +145,7 @@ public QueueInfo getQueueInfo( queueInfo.setCapacity(100.0f); queueInfo.setMaximumCapacity(100.0f); queueInfo.setChildQueues(new ArrayList()); + queueInfo.setQueueState(QueueState.RUNNING); return queueInfo; } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DefaultSchedulerPage.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DefaultSchedulerPage.java index a0a3030be3..a232e5bcc8 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DefaultSchedulerPage.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DefaultSchedulerPage.java @@ -18,20 +18,177 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; -import org.apache.hadoop.yarn.webapp.SubView; -import org.apache.hadoop.yarn.webapp.view.HtmlBlock; +import com.google.inject.Inject; +import com.google.inject.servlet.RequestScoped; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; +import org.apache.hadoop.yarn.webapp.SubView; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.*; +import org.apache.hadoop.yarn.webapp.view.HtmlBlock; +import org.apache.hadoop.yarn.api.records.QueueInfo; +import org.apache.hadoop.yarn.api.records.QueueState; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; +import org.apache.hadoop.yarn.webapp.view.InfoBlock; + +import static org.apache.hadoop.yarn.util.StringHelper.*; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.*; class DefaultSchedulerPage extends RmView { + static final String _Q = ".ui-state-default.ui-corner-all"; + static final float WIDTH_F = 0.8f; + static final String Q_END = "left:101%"; + static final String OVER = "font-size:1px;background:rgba(255, 140, 0, 0.8)"; + static final String UNDER = "font-size:1px;background:rgba(50, 205, 50, 0.8)"; + static final float EPSILON = 1e-8f; + + static class QueueInfoBlock extends HtmlBlock { + final RMContext rmContext; + final FifoScheduler fs; + final String qName; + final QueueInfo qInfo; + + @Inject QueueInfoBlock(RMContext context, ViewContext ctx, ResourceManager rm) { + super(ctx); + this.rmContext = context; + + fs = (FifoScheduler) rm.getResourceScheduler(); + qName = fs.getQueueInfo("",false,false).getQueueName(); + qInfo = fs.getQueueInfo(qName,true,true); + } - static class QueueBlock extends HtmlBlock { @Override public void render(Block html) { - html.h2("Under construction"); + String minmemoryresource = + Integer.toString(fs.getMinimumResourceCapability().getMemory()); + String maxmemoryresource = + Integer.toString(fs.getMaximumResourceCapability().getMemory()); + String qstate = (qInfo.getQueueState() == QueueState.RUNNING) ? + "Running" : + (qInfo.getQueueState() == QueueState.STOPPED) ? + "Stopped" : "Unknown"; + + int usedNodeMem = 0; + int availNodeMem = 0; + int totNodeMem = 0; + int nodeContainers = 0; + + for (RMNode ni : this.rmContext.getRMNodes().values()) { + usedNodeMem += fs.getUsedResource(ni.getNodeID()).getMemory(); + availNodeMem += fs.getAvailableResource(ni.getNodeID()).getMemory(); + totNodeMem += ni.getTotalCapability().getMemory(); + nodeContainers += fs.getNodeReport(ni.getNodeID()).getNumContainers(); + } + + info("\'" + qName + "\' Queue Status"). + _("Queue State:" , qstate). + _("Minimum Queue Memory Capacity:" , minmemoryresource). + _("Maximum Queue Memory Capacity:" , maxmemoryresource). + _("Number of Nodes:" , Integer.toString(this.rmContext.getRMNodes().size())). + _("Used Node Capacity:" , Integer.toString(usedNodeMem)). + _("Available Node Capacity:" , Integer.toString(availNodeMem)). + _("Total Node Capacity:" , Integer.toString(totNodeMem)). + _("Number of Node Containers:" , Integer.toString(nodeContainers)); + + html._(InfoBlock.class); } } + static class QueuesBlock extends HtmlBlock { + final FifoScheduler fs; + final String qName; + final QueueInfo qInfo; + + @Inject QueuesBlock(ResourceManager rm) { + fs = (FifoScheduler) rm.getResourceScheduler(); + qName = fs.getQueueInfo("",false,false).getQueueName(); + qInfo = fs.getQueueInfo(qName,false,false); + } + + @Override + public void render(Block html) { + UL>> ul = html. + div("#cs-wrapper.ui-widget"). + div(".ui-widget-header.ui-corner-top"). + _("FifoScheduler Queue")._(). + div("#cs.ui-widget-content.ui-corner-bottom"). + ul(); + + if (fs == null) { + ul. + li(). + a(_Q).$style(width(WIDTH_F)). + span().$style(Q_END)._("100% ")._(). + span(".q", "default")._()._(); + } else { + float used = qInfo.getCurrentCapacity() / 100.0f; + float set = qInfo.getCapacity() / 100.0f; + float delta = Math.abs(set - used) + 0.001f; + ul. + li(). + a(_Q).$style(width(WIDTH_F)). + $title(join("used:", percent(used))). + span().$style(Q_END)._("100%")._(). + span().$style(join(width(delta), ';', used > set ? OVER : UNDER, + ';', used > set ? left(set) : left(used)))._(".")._(). + span(".q", qName)._(). + _(QueueInfoBlock.class)._(); + } + + ul._()._(). + script().$type("text/javascript"). + _("$('#cs').hide();")._()._(). + _(AppsBlock.class); + } + } + + + @Override protected void postHead(Page.HTML<_> html) { + html. + style().$type("text/css"). + _("#cs { padding: 0.5em 0 1em 0; margin-bottom: 1em; position: relative }", + "#cs ul { list-style: none }", + "#cs a { font-weight: normal; margin: 2px; position: relative }", + "#cs a span { font-weight: normal; font-size: 80% }", + "#cs-wrapper .ui-widget-header { padding: 0.2em 0.5em }", + "table.info tr th {width: 50%}")._(). // to center info table + script("/static/jt/jquery.jstree.js"). + script().$type("text/javascript"). + _("$(function() {", + " $('#cs a span').addClass('ui-corner-all').css('position', 'absolute');", + " $('#cs').bind('loaded.jstree', function (e, data) {", + " data.inst.open_all(); }).", + " jstree({", + " core: { animation: 188, html_titles: true },", + " plugins: ['themeroller', 'html_data', 'ui'],", + " themeroller: { item_open: 'ui-icon-minus',", + " item_clsd: 'ui-icon-plus', item_leaf: 'ui-icon-gear'", + " }", + " });", + " $('#cs').bind('select_node.jstree', function(e, data) {", + " var q = $('.q', data.rslt.obj).first().text();", + " if (q == 'root') q = '';", + " $('#apps').dataTable().fnFilter(q, 3);", + " });", + " $('#cs').show();", + "});")._(); + } + @Override protected Class content() { - return QueueBlock.class; + return QueuesBlock.class; + } + + static String percent(float f) { + return String.format("%.1f%%", f * 100); + } + + static String width(float f) { + return String.format("width:%.1f%%", f * 100); + } + + static String left(float f) { + return String.format("left:%.1f%%", f * 100); } } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java index 8ea9e80762..3dfa301e34 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java @@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; import org.apache.hadoop.yarn.webapp.WebApps; import org.apache.hadoop.yarn.webapp.test.WebAppTests; import org.junit.Test; @@ -168,9 +169,38 @@ static void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { conf.setCapacity(C13, 40); } + public static ResourceManager mockFifoRm(int apps, int racks, int nodes, + int mbsPerNode) + throws Exception { + ResourceManager rm = mock(ResourceManager.class); + RMContext rmContext = mockRMContext(apps, racks, nodes, + mbsPerNode); + ResourceScheduler rs = mockFifoScheduler(); + when(rm.getResourceScheduler()).thenReturn(rs); + when(rm.getRMContext()).thenReturn(rmContext); + return rm; + } + + public static FifoScheduler mockFifoScheduler() throws Exception { + CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + setupFifoQueueConfiguration(conf); + + FifoScheduler fs = new FifoScheduler(); + fs.reinitialize(conf, null, null); + return fs; + } + + static void setupFifoQueueConfiguration(CapacitySchedulerConfiguration conf) { + // Define default queue + conf.setQueues("default", new String[] {"default"}); + conf.setCapacity("default", 100); + } + public static void main(String[] args) throws Exception { // For manual testing WebApps.$for("yarn", new TestRMWebApp()).at(8888).inDevMode(). start(new RMWebApp(mockRm(101, 8, 8, 8*GiB))).joinThread(); + WebApps.$for("yarn", new TestRMWebApp()).at(8888).inDevMode(). + start(new RMWebApp(mockFifoRm(10, 1, 4, 8*GiB))).joinThread(); } }