big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-2690. Web-page for FifoScheduler. Contributed by Eric Payne.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1166958 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Arun Murthy 2011-09-09 02:04:55 +00:00
parent fafe8cd28e
commit 035937bee8
4 changed files with 196 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-mapreduce-project/CHANGES.txt
View File

@ -255,6 +255,8 @@ Release 0.23.0 - Unreleased
MAPREDUCE-2864. Normalize configuration variable names for YARN. (Robert
Evans via acmurthy)
MAPREDUCE-2690. Web-page for FifoScheduler. (Eric Payne via acmurthy)
OPTIMIZATIONS
MAPREDUCE-2026. Make JobTracker.getJobCounters() and

2
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
View File

@ -82,6 +82,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.api.records.QueueState;
@LimitedPrivate("yarn")
@Evolving
@ -144,6 +145,7 @@ public QueueInfo getQueueInfo(
queueInfo.setCapacity(100.0f);
queueInfo.setMaximumCapacity(100.0f);
queueInfo.setChildQueues(new ArrayList<QueueInfo>());
queueInfo.setQueueState(QueueState.RUNNING);
return queueInfo;
}

167
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DefaultSchedulerPage.java
View File

@ -18,20 +18,177 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import com.google.inject.Inject;
import com.google.inject.servlet.RequestScoped;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.*;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
import static org.apache.hadoop.yarn.util.StringHelper.*;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.*;
class DefaultSchedulerPage extends RmView {
static final String _Q = ".ui-state-default.ui-corner-all";
static final float WIDTH_F = 0.8f;
static final String Q_END = "left:101%";
static final String OVER = "font-size:1px;background:rgba(255, 140, 0, 0.8)";
static final String UNDER = "font-size:1px;background:rgba(50, 205, 50, 0.8)";
static final float EPSILON = 1e-8f;
static class QueueInfoBlock extends HtmlBlock {
final RMContext rmContext;
final FifoScheduler fs;
final String qName;
final QueueInfo qInfo;
@Inject QueueInfoBlock(RMContext context, ViewContext ctx, ResourceManager rm) {
super(ctx);
this.rmContext = context;
fs = (FifoScheduler) rm.getResourceScheduler();
qName = fs.getQueueInfo("",false,false).getQueueName();
qInfo = fs.getQueueInfo(qName,true,true);
}
static class QueueBlock extends HtmlBlock {
@Override public void render(Block html) {
html.h2("Under construction");
String minmemoryresource =
Integer.toString(fs.getMinimumResourceCapability().getMemory());
String maxmemoryresource =
Integer.toString(fs.getMaximumResourceCapability().getMemory());
String qstate = (qInfo.getQueueState() == QueueState.RUNNING) ?
"Running" :
(qInfo.getQueueState() == QueueState.STOPPED) ?
"Stopped" : "Unknown";
int usedNodeMem = 0;
int availNodeMem = 0;
int totNodeMem = 0;
int nodeContainers = 0;
for (RMNode ni : this.rmContext.getRMNodes().values()) {
usedNodeMem += fs.getUsedResource(ni.getNodeID()).getMemory();
availNodeMem += fs.getAvailableResource(ni.getNodeID()).getMemory();
totNodeMem += ni.getTotalCapability().getMemory();
nodeContainers += fs.getNodeReport(ni.getNodeID()).getNumContainers();
}
info("\'" + qName + "\' Queue Status").
_("Queue State:" , qstate).
_("Minimum Queue Memory Capacity:" , minmemoryresource).
_("Maximum Queue Memory Capacity:" , maxmemoryresource).
_("Number of Nodes:" , Integer.toString(this.rmContext.getRMNodes().size())).
_("Used Node Capacity:" , Integer.toString(usedNodeMem)).
_("Available Node Capacity:" , Integer.toString(availNodeMem)).
_("Total Node Capacity:" , Integer.toString(totNodeMem)).
_("Number of Node Containers:" , Integer.toString(nodeContainers));
html._(InfoBlock.class);
}
}
static class QueuesBlock extends HtmlBlock {
final FifoScheduler fs;
final String qName;
final QueueInfo qInfo;
@Inject QueuesBlock(ResourceManager rm) {
fs = (FifoScheduler) rm.getResourceScheduler();
qName = fs.getQueueInfo("",false,false).getQueueName();
qInfo = fs.getQueueInfo(qName,false,false);
}
@Override
public void render(Block html) {
UL<DIV<DIV<Hamlet>>> ul = html.
div("#cs-wrapper.ui-widget").
div(".ui-widget-header.ui-corner-top").
_("FifoScheduler Queue")._().
div("#cs.ui-widget-content.ui-corner-bottom").
ul();
if (fs == null) {
ul.
li().
a(_Q).$style(width(WIDTH_F)).
span().$style(Q_END)._("100% ")._().
span(".q", "default")._()._();
} else {
float used = qInfo.getCurrentCapacity() / 100.0f;
float set = qInfo.getCapacity() / 100.0f;
float delta = Math.abs(set - used) + 0.001f;
ul.
li().
a(_Q).$style(width(WIDTH_F)).
$title(join("used:", percent(used))).
span().$style(Q_END)._("100%")._().
span().$style(join(width(delta), ';', used > set ? OVER : UNDER,
';', used > set ? left(set) : left(used)))._(".")._().
span(".q", qName)._().
_(QueueInfoBlock.class)._();
}
ul._()._().
script().$type("text/javascript").
_("$('#cs').hide();")._()._().
_(AppsBlock.class);
}
}
@Override protected void postHead(Page.HTML<_> html) {
html.
style().$type("text/css").
_("#cs { padding: 0.5em 0 1em 0; margin-bottom: 1em; position: relative }",
"#cs ul { list-style: none }",
"#cs a { font-weight: normal; margin: 2px; position: relative }",
"#cs a span { font-weight: normal; font-size: 80% }",
"#cs-wrapper .ui-widget-header { padding: 0.2em 0.5em }",
"table.info tr th {width: 50%}")._(). // to center info table
script("/static/jt/jquery.jstree.js").
script().$type("text/javascript").
_("$(function() {",
" $('#cs a span').addClass('ui-corner-all').css('position', 'absolute');",
" $('#cs').bind('loaded.jstree', function (e, data) {",
" data.inst.open_all(); }).",
" jstree({",
" core: { animation: 188, html_titles: true },",
" plugins: ['themeroller', 'html_data', 'ui'],",
" themeroller: { item_open: 'ui-icon-minus',",
" item_clsd: 'ui-icon-plus', item_leaf: 'ui-icon-gear'",
" }",
" });",
" $('#cs').bind('select_node.jstree', function(e, data) {",
" var q = $('.q', data.rslt.obj).first().text();",
" if (q == 'root') q = '';",
" $('#apps').dataTable().fnFilter(q, 3);",
" });",
" $('#cs').show();",
"});")._();
}
@Override protected Class<? extends SubView> content() {
return QueueBlock.class;
return QueuesBlock.class;
}
static String percent(float f) {
return String.format("%.1f%%", f * 100);
}
static String width(float f) {
return String.format("width:%.1f%%", f * 100);
}
static String left(float f) {
return String.format("left:%.1f%%", f * 100);
}
}

30
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
View File

@ -42,6 +42,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.webapp.WebApps;
import org.apache.hadoop.yarn.webapp.test.WebAppTests;
import org.junit.Test;
@ -168,9 +169,38 @@ static void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
conf.setCapacity(C13, 40);
}
public static ResourceManager mockFifoRm(int apps, int racks, int nodes,
int mbsPerNode)
throws Exception {
ResourceManager rm = mock(ResourceManager.class);
RMContext rmContext = mockRMContext(apps, racks, nodes,
mbsPerNode);
ResourceScheduler rs = mockFifoScheduler();
when(rm.getResourceScheduler()).thenReturn(rs);
when(rm.getRMContext()).thenReturn(rmContext);
return rm;
}
public static FifoScheduler mockFifoScheduler() throws Exception {
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
setupFifoQueueConfiguration(conf);
FifoScheduler fs = new FifoScheduler();
fs.reinitialize(conf, null, null);
return fs;
}
static void setupFifoQueueConfiguration(CapacitySchedulerConfiguration conf) {
// Define default queue
conf.setQueues("default", new String[] {"default"});
conf.setCapacity("default", 100);
}
public static void main(String[] args) throws Exception {
// For manual testing
WebApps.$for("yarn", new TestRMWebApp()).at(8888).inDevMode().
start(new RMWebApp(mockRm(101, 8, 8, 8*GiB))).joinThread();
WebApps.$for("yarn", new TestRMWebApp()).at(8888).inDevMode().
start(new RMWebApp(mockFifoRm(10, 1, 4, 8*GiB))).joinThread();
}
}