diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 754b622f93..fee3fc9fab 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -473,6 +473,9 @@ Release 0.23.1 - Unreleased MAPREDUCE-3652. org.apache.hadoop.mapred.TestWebUIAuthorization.testWebUIAuthorization fails. (Thomas Graves via mahadev) + MAPREDUCE-3625. CapacityScheduler web-ui display of queue's used capacity is broken. + (Jason Lowe via mahadev) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java index 467b4d33de..edf0231ae3 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java @@ -42,11 +42,12 @@ class CapacitySchedulerPage extends RmView { static final String _Q = ".ui-state-default.ui-corner-all"; - static final float WIDTH_F = 0.8f; + static final float Q_MAX_WIDTH = 0.8f; + static final float Q_STATS_POS = Q_MAX_WIDTH + 0.05f; static final String Q_END = "left:101%"; - static final String OVER = "font-size:1px;background:rgba(255, 140, 0, 0.8)"; - static final String UNDER = "font-size:1px;background:rgba(50, 205, 50, 0.8)"; - static final float EPSILON = 1e-8f; + static final String Q_GIVEN = "left:0%;background:none;border:1px dashed rgba(0,0,0,0.25)"; + static final String Q_OVER = "background:rgba(255, 140, 0, 0.8)"; + static final String Q_UNDER = "background:rgba(50, 205, 50, 0.8)"; @RequestScoped static class CSQInfo { @@ -106,18 +107,20 @@ public void render(Block html) { for (CapacitySchedulerQueueInfo info : subQueues) { float used = info.getUsedCapacity() / 100; float set = info.getCapacity() / 100; - float delta = Math.abs(set - used) + 0.001f; float max = info.getMaxCapacity() / 100; LI> li = ul. li(). - a(_Q).$style(width(max * WIDTH_F)). - $title(join("used:", percent(used), " set:", percent(set), - " max:", percent(max))). - //span().$style(Q_END)._(absMaxPct)._(). - span().$style(join(width(delta/max), ';', - used > set ? OVER : UNDER, ';', - used > set ? left(set/max) : left(used/max)))._('.')._(). - span(".q", info.getQueuePath().substring(5))._(); + a(_Q).$style(width(max * Q_MAX_WIDTH)). + $title(join("capacity:", percent(set), " used:", percent(used), + " max capacity:", percent(max))). + span().$style(join(Q_GIVEN, ";font-size:1px;", width(set/max))). + _('.')._(). + span().$style(join(width(used*set/max), + ";font-size:1px;left:0%;", used > 1 ? Q_OVER : Q_UNDER)). + _('.')._(). + span(".q", info.getQueuePath().substring(5))._(). + span().$class("qstats").$style(left(Q_STATS_POS)). + _(join(percent(used), " used"))._(); csqinfo.qinfo = info; if (info.getSubQueues() == null) { @@ -153,7 +156,7 @@ public void render(Block html) { if (cs == null) { ul. li(). - a(_Q).$style(width(WIDTH_F)). + a(_Q).$style(width(Q_MAX_WIDTH)). span().$style(Q_END)._("100% ")._(). span(".q", "default")._()._(); } else { @@ -163,16 +166,26 @@ public void render(Block html) { csqinfo.qinfo = null; float used = sinfo.getUsedCapacity() / 100; - float set = sinfo.getCapacity() / 100; - float delta = Math.abs(set - used) + 0.001f; ul. + li().$style("margin-bottom: 1em"). + span().$style("font-weight: bold")._("Legend:")._(). + span().$class("qlegend ui-corner-all").$style(Q_GIVEN). + _("Capacity")._(). + span().$class("qlegend ui-corner-all").$style(Q_UNDER). + _("Used")._(). + span().$class("qlegend ui-corner-all").$style(Q_OVER). + _("Used (over capacity)")._(). + span().$class("qlegend ui-corner-all ui-state-default"). + _("Max Capacity")._(). + _(). li(). - a(_Q).$style(width(WIDTH_F)). + a(_Q).$style(width(Q_MAX_WIDTH)). $title(join("used:", percent(used))). - span().$style(Q_END)._("100%")._(). - span().$style(join(width(delta), ';', used > set ? OVER : UNDER, - ';', used > set ? left(set) : left(used)))._(".")._(). + span().$style(join(width(used), ";left:0%;", + used > 1 ? Q_OVER : Q_UNDER))._(".")._(). span(".q", "root")._(). + span().$class("qstats").$style(left(Q_STATS_POS)). + _(join(percent(used), " used"))._(). _(QueueBlock.class)._(); } ul._()._(). @@ -190,6 +203,8 @@ public void render(Block html) { "#cs a { font-weight: normal; margin: 2px; position: relative }", "#cs a span { font-weight: normal; font-size: 80% }", "#cs-wrapper .ui-widget-header { padding: 0.2em 0.5em }", + ".qstats { font-weight: normal; font-size: 80%; position: absolute }", + ".qlegend { font-weight: normal; padding: 0 1em; margin: 1em }", "table.info tr th {width: 50%}")._(). // to center info table script("/static/jt/jquery.jstree.js"). script().$type("text/javascript"). diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index dd9ab16b2d..edce4469c9 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -25,6 +25,7 @@ import javax.xml.bind.annotation.XmlSeeAlso; import javax.xml.bind.annotation.XmlTransient; +import org.apache.hadoop.yarn.api.records.QueueState; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; @XmlRootElement @@ -47,7 +48,7 @@ public class CapacitySchedulerQueueInfo { protected int numApplications; protected String usedResources; protected String queueName; - protected String state; + protected QueueState state; protected ArrayList subQueues; CapacitySchedulerQueueInfo() { @@ -69,7 +70,7 @@ public class CapacitySchedulerQueueInfo { numApplications = q.getNumApplications(); usedResources = q.getUsedResources().toString(); queueName = q.getQueueName(); - state = q.getState().toString(); + state = q.getState(); } public float getCapacity() { @@ -109,7 +110,7 @@ public String getQueueName() { } public String getQueueState() { - return this.state; + return this.state.toString(); } public String getQueuePath() { diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index 3ee0dac104..4c6ea02b53 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -62,6 +62,31 @@ public class TestRMWebServicesCapacitySched extends JerseyTest { private static MockRM rm; private CapacitySchedulerConfiguration csConf; + private class QueueInfo { + float capacity; + float usedCapacity; + float maxCapacity; + float absoluteCapacity; + float absoluteMaxCapacity; + float utilization; + int numApplications; + String usedResources; + String queueName; + String state; + } + + private class LeafQueueInfo extends QueueInfo { + int numActiveApplications; + int numPendingApplications; + int numContainers; + int maxApplications; + int maxApplicationsPerUser; + int maxActiveApplications; + int maxActiveApplicationsPerUser; + int userLimit; + float userLimitFactor; + } + private Injector injector = Guice.createInjector(new ServletModule() { @Override protected void configureServlets() { @@ -217,29 +242,51 @@ public void verifyClusterSchedulerXML(NodeList nodes) throws Exception { public void verifySubQueueXML(Element qElem, String q, float parentAbsCapacity) throws Exception { - float absCapacity = WebServicesTestUtils.getXmlFloat(qElem, "absoluteCapacity"); - verifySubQueueGeneric(q, - WebServicesTestUtils.getXmlFloat(qElem, "usedCapacity"), - WebServicesTestUtils.getXmlFloat(qElem, "capacity"), - WebServicesTestUtils.getXmlFloat(qElem, "maxCapacity"), - absCapacity, - WebServicesTestUtils.getXmlFloat(qElem, "absoluteMaxCapacity"), - parentAbsCapacity, - WebServicesTestUtils.getXmlString(qElem, "queueName"), - WebServicesTestUtils.getXmlString(qElem, "state")); - NodeList queues = qElem.getElementsByTagName("subQueues"); + QueueInfo qi = (queues != null) ? new QueueInfo() : new LeafQueueInfo(); + qi.capacity = WebServicesTestUtils.getXmlFloat(qElem, "capacity"); + qi.usedCapacity = + WebServicesTestUtils.getXmlFloat(qElem, "usedCapacity"); + qi.maxCapacity = WebServicesTestUtils.getXmlFloat(qElem, "maxCapacity"); + qi.absoluteCapacity = WebServicesTestUtils.getXmlFloat(qElem, "absoluteCapacity"); + qi.absoluteMaxCapacity = + WebServicesTestUtils.getXmlFloat(qElem, "absoluteMaxCapacity"); + qi.utilization = WebServicesTestUtils.getXmlFloat(qElem, "utilization"); + qi.numApplications = + WebServicesTestUtils.getXmlInt(qElem, "numApplications"); + qi.usedResources = + WebServicesTestUtils.getXmlString(qElem, "usedResources"); + qi.queueName = WebServicesTestUtils.getXmlString(qElem, "queueName"); + qi.state = WebServicesTestUtils.getXmlString(qElem, "state"); + verifySubQueueGeneric(q, qi, parentAbsCapacity); + if (queues != null) { for (int j = 0; j < queues.getLength(); j++) { Element subqElem = (Element) queues.item(j); String qName = WebServicesTestUtils.getXmlString(subqElem, "queueName"); String q2 = q + "." + qName; - verifySubQueueXML(subqElem, q2, absCapacity); + verifySubQueueXML(subqElem, q2, qi.absoluteCapacity); } } else { - verifyLeafQueueGeneric(q, - WebServicesTestUtils.getXmlInt(qElem, "userLimit"), - WebServicesTestUtils.getXmlFloat(qElem, "userLimitFactor")); + LeafQueueInfo lqi = (LeafQueueInfo) qi; + lqi.numActiveApplications = + WebServicesTestUtils.getXmlInt(qElem, "numActiveApplications"); + lqi.numPendingApplications = + WebServicesTestUtils.getXmlInt(qElem, "numPendingApplications"); + lqi.numContainers = + WebServicesTestUtils.getXmlInt(qElem, "numContainers"); + lqi.maxApplications = + WebServicesTestUtils.getXmlInt(qElem, "maxApplications"); + lqi.maxApplicationsPerUser = + WebServicesTestUtils.getXmlInt(qElem, "maxApplicationsPerUser"); + lqi.maxActiveApplications = + WebServicesTestUtils.getXmlInt(qElem, "maxActiveApplications"); + lqi.maxActiveApplicationsPerUser = + WebServicesTestUtils.getXmlInt(qElem, "maxActiveApplicationsPerUser"); + lqi.userLimit = WebServicesTestUtils.getXmlInt(qElem, "userLimit"); + lqi.userLimitFactor = + WebServicesTestUtils.getXmlFloat(qElem, "userLimitFactor"); + verifyLeafQueueGeneric(q, lqi); } } @@ -286,16 +333,19 @@ private void verifySubQueue(JSONObject info, String q, float parentAbsCapacity) } assertEquals("incorrect number of elements", numExpectedElements, info.length()); - float absCapacity = (float) info.getDouble("absoluteCapacity"); + QueueInfo qi = isParentQueue ? new QueueInfo() : new LeafQueueInfo(); + qi.capacity = (float) info.getDouble("capacity"); + qi.usedCapacity = (float) info.getDouble("usedCapacity"); + qi.maxCapacity = (float) info.getDouble("maxCapacity"); + qi.absoluteCapacity = (float) info.getDouble("absoluteCapacity"); + qi.absoluteMaxCapacity = (float) info.getDouble("absoluteMaxCapacity"); + qi.utilization = (float) info.getDouble("utilization"); + qi.numApplications = info.getInt("numApplications"); + qi.usedResources = info.getString("usedResources"); + qi.queueName = info.getString("queueName"); + qi.state = info.getString("state"); - verifySubQueueGeneric(q, (float) info.getDouble("usedCapacity"), - (float) info.getDouble("capacity"), - (float) info.getDouble("maxCapacity"), - absCapacity, - (float) info.getDouble("absoluteMaxCapacity"), - parentAbsCapacity, - info.getString("queueName"), - info.getString("state")); + verifySubQueueGeneric(q, qi, parentAbsCapacity); if (isParentQueue) { JSONArray arr = info.getJSONArray("subQueues"); @@ -303,50 +353,84 @@ private void verifySubQueue(JSONObject info, String q, float parentAbsCapacity) for (int i = 0; i < arr.length(); i++) { JSONObject obj = arr.getJSONObject(i); String q2 = q + "." + obj.getString("queueName"); - verifySubQueue(obj, q2, absCapacity); + verifySubQueue(obj, q2, qi.absoluteCapacity); } } else { - verifyLeafQueueGeneric(q, info.getInt("userLimit"), - (float) info.getDouble("userLimitFactor")); + LeafQueueInfo lqi = (LeafQueueInfo) qi; + lqi.numActiveApplications = info.getInt("numActiveApplications"); + lqi.numPendingApplications = info.getInt("numPendingApplications"); + lqi.numContainers = info.getInt("numContainers"); + lqi.maxApplications = info.getInt("maxApplications"); + lqi.maxApplicationsPerUser = info.getInt("maxApplicationsPerUser"); + lqi.maxActiveApplications = info.getInt("maxActiveApplications"); + lqi.maxActiveApplicationsPerUser = info.getInt("maxActiveApplicationsPerUser"); + lqi.userLimit = info.getInt("userLimit"); + lqi.userLimitFactor = (float) info.getDouble("userLimitFactor"); + verifyLeafQueueGeneric(q, lqi); } } - private void verifySubQueueGeneric(String q, float usedCapacity, - float capacity, float maxCapacity, - float absCapacity, float absMaxCapacity, - float parentAbsCapacity, - String qname, String state) - throws Exception { + private void verifySubQueueGeneric(String q, QueueInfo info, + float parentAbsCapacity) throws Exception { String[] qArr = q.split("\\."); assertTrue("q name invalid: " + q, qArr.length > 1); String qshortName = qArr[qArr.length - 1]; - assertEquals("usedCapacity doesn't match", 0, usedCapacity, 1e-3f); - assertEquals("capacity doesn't match", csConf.getCapacity(q), capacity, - 1e-3f); + assertEquals("usedCapacity doesn't match", 0, info.usedCapacity, 1e-3f); + assertEquals("capacity doesn't match", csConf.getCapacity(q), + info.capacity, 1e-3f); float expectCapacity = csConf.getMaximumCapacity(q); - float expectAbsMaxCapacity = parentAbsCapacity * (maxCapacity/100); + float expectAbsMaxCapacity = parentAbsCapacity * (info.maxCapacity/100); if (CapacitySchedulerConfiguration.UNDEFINED == expectCapacity) { expectCapacity = 100; expectAbsMaxCapacity = 100; } - assertEquals("maxCapacity doesn't match", expectCapacity, maxCapacity, - 1e-3f); + assertEquals("maxCapacity doesn't match", expectCapacity, + info.maxCapacity, 1e-3f); assertEquals("absoluteCapacity doesn't match", - parentAbsCapacity * (capacity/100), absCapacity, 1e-3f); + parentAbsCapacity * (info.capacity/100), info.absoluteCapacity, 1e-3f); assertEquals("absoluteMaxCapacity doesn't match", - expectAbsMaxCapacity, absMaxCapacity, 1e-3f); - assertTrue("queueName doesn't match, got: " + qname + " expected: " + q, - qshortName.matches(qname)); + expectAbsMaxCapacity, info.absoluteMaxCapacity, 1e-3f); + assertEquals("utilization doesn't match", 0, info.utilization, 1e-3f); + assertEquals("numApplications doesn't match", 0, info.numApplications); + assertTrue("usedResources doesn't match", + info.usedResources.matches("memory: 0")); + assertTrue("queueName doesn't match, got: " + info.queueName + + " expected: " + q, qshortName.matches(info.queueName)); assertTrue("state doesn't match", - (csConf.getState(q).toString()).matches(state)); + (csConf.getState(q).toString()).matches(info.state)); } - private void verifyLeafQueueGeneric(String q, int userLimit, - float userLimitFactor) throws Exception { - assertEquals("userLimit doesn't match", csConf.getUserLimit(q), userLimit); + private void verifyLeafQueueGeneric(String q, LeafQueueInfo info) + throws Exception { + assertEquals("numActiveApplications doesn't match", + 0, info.numActiveApplications); + assertEquals("numPendingApplications doesn't match", + 0, info.numPendingApplications); + assertEquals("numContainers doesn't match", + 0, info.numContainers); + + int maxSystemApps = csConf.getMaximumSystemApplications(); + int expectedMaxApps = (int)(maxSystemApps * (info.absoluteCapacity/100)); + int expectedMaxAppsPerUser = + (int)(expectedMaxApps * (info.userLimit/100.0f) * info.userLimitFactor); + + // TODO: would like to use integer comparisons here but can't due to + // roundoff errors in absolute capacity calculations + assertEquals("maxApplications doesn't match", + (float)expectedMaxApps, (float)info.maxApplications, 1.0f); + assertEquals("maxApplicationsPerUser doesn't match", + (float)expectedMaxAppsPerUser, + (float)info.maxApplicationsPerUser, info.userLimitFactor); + + assertTrue("maxActiveApplications doesn't match", + info.maxActiveApplications > 0); + assertTrue("maxActiveApplicationsPerUser doesn't match", + info.maxActiveApplicationsPerUser > 0); + assertEquals("userLimit doesn't match", csConf.getUserLimit(q), + info.userLimit); assertEquals("userLimitFactor doesn't match", - csConf.getUserLimitFactor(q), userLimitFactor, 1e-3f); + csConf.getUserLimitFactor(q), info.userLimitFactor, 1e-3f); } }