From 746ad6e989683fe1dfc61a611702c9be7b5cd264 Mon Sep 17 00:00:00 2001 From: Jian He Date: Tue, 30 Dec 2014 16:46:26 -0800 Subject: [PATCH] Revert "YARN-2492(wrong jira number). Added node-labels page on RM web UI. Contributed by Wangda Tan" This reverts commit 5f57b904f550515693d93a2959e663b0d0260696. --- hadoop-yarn-project/CHANGES.txt | 2 - .../nodelabels/CommonNodeLabelsManager.java | 21 ++- .../hadoop/yarn/nodelabels/NodeLabel.java | 96 ------------ .../hadoop/yarn/webapp/YarnWebParams.java | 1 - .../nodelabels/RMNodeLabelsManager.java | 36 +---- .../resourcemanager/webapp/NavBlock.java | 3 +- .../webapp/NodeLabelsPage.java | 91 ------------ .../resourcemanager/webapp/NodesPage.java | 139 +++++++++--------- .../resourcemanager/webapp/RMWebApp.java | 1 - .../resourcemanager/webapp/RmController.java | 6 +- .../server/resourcemanager/MockNodes.java | 46 +----- .../TestWorkPreservingRMRestart.java | 2 +- .../nodelabels/TestRMNodeLabelsManager.java | 33 ----- .../resourcemanager/webapp/TestNodesPage.java | 45 ------ 14 files changed, 101 insertions(+), 421 deletions(-) delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabel.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeLabelsPage.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 0ebf8a3351..e707ea7953 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -155,8 +155,6 @@ Release 2.7.0 - UNRELEASED YARN-2993. Several fixes (missing acl check, error log msg ...) and some refinement in AdminService. (Yi Liu via junping_du) - YARN-2943. Added node-labels page on RM web UI. (Wangda Tan via jianhe) - OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java index e888cc56d8..070aa1fc85 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java @@ -72,8 +72,8 @@ public class CommonNodeLabelsManager extends AbstractService { protected Dispatcher dispatcher; - protected ConcurrentMap labelCollections = - new ConcurrentHashMap(); + protected ConcurrentMap labelCollections = + new ConcurrentHashMap(); protected ConcurrentMap nodeCollections = new ConcurrentHashMap(); @@ -82,6 +82,19 @@ public class CommonNodeLabelsManager extends AbstractService { protected NodeLabelsStore store; + protected static class Label { + private Resource resource; + + protected Label() { + this.resource = Resource.newInstance(0, 0); + } + + public Resource getResource() { + return this.resource; + } + + } + /** * A Host can have multiple Nodes */ @@ -188,7 +201,7 @@ public class CommonNodeLabelsManager extends AbstractService { protected void serviceInit(Configuration conf) throws Exception { initNodeLabelStore(conf); - labelCollections.put(NO_LABEL, new NodeLabel(NO_LABEL)); + labelCollections.put(NO_LABEL, new Label()); } protected void initNodeLabelStore(Configuration conf) throws Exception { @@ -258,7 +271,7 @@ public class CommonNodeLabelsManager extends AbstractService { for (String label : labels) { // shouldn't overwrite it to avoid changing the Label.resource if (this.labelCollections.get(label) == null) { - this.labelCollections.put(label, new NodeLabel(label)); + this.labelCollections.put(label, new Label()); newLabels.add(label); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabel.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabel.java deleted file mode 100644 index 766864887d..0000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabel.java +++ /dev/null @@ -1,96 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.nodelabels; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.util.resource.Resources; - -public class NodeLabel implements Comparable { - private Resource resource; - private int numActiveNMs; - private String labelName; - - public NodeLabel(String labelName) { - this(labelName, Resource.newInstance(0, 0), 0); - } - - protected NodeLabel(String labelName, Resource res, int activeNMs) { - this.labelName = labelName; - this.resource = res; - this.numActiveNMs = activeNMs; - } - - public void addNode(Resource nodeRes) { - Resources.addTo(resource, nodeRes); - numActiveNMs++; - } - - public void removeNode(Resource nodeRes) { - Resources.subtractFrom(resource, nodeRes); - numActiveNMs--; - } - - public Resource getResource() { - return this.resource; - } - - public int getNumActiveNMs() { - return numActiveNMs; - } - - public String getLabelName() { - return labelName; - } - - public NodeLabel getCopy() { - return new NodeLabel(labelName, resource, numActiveNMs); - } - - @Override - public int compareTo(NodeLabel o) { - // We should always put empty label entry first after sorting - if (labelName.isEmpty() != o.getLabelName().isEmpty()) { - if (labelName.isEmpty()) { - return -1; - } - return 1; - } - - return labelName.compareTo(o.getLabelName()); - } - - @Override - public boolean equals(Object obj) { - if (obj instanceof NodeLabel) { - NodeLabel other = (NodeLabel) obj; - return Resources.equals(resource, other.getResource()) - && StringUtils.equals(labelName, other.getLabelName()) - && (other.getNumActiveNMs() == numActiveNMs); - } - return false; - } - - @Override - public int hashCode() { - final int prime = 502357; - return (int) ((((long) labelName.hashCode() << 8) - + (resource.hashCode() << 4) + numActiveNMs) % prime); - } -} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java index 62c3c7a0ef..91d2a2019a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java @@ -32,5 +32,4 @@ public interface YarnWebParams { String APP_STATE = "app.state"; String QUEUE_NAME = "queue.name"; String NODE_STATE = "node.state"; - String NODE_LABEL = "node.label"; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java index 828d1bc34d..646441a10f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java @@ -19,12 +19,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.nodelabels; import java.io.IOException; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; @@ -39,7 +37,6 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; -import org.apache.hadoop.yarn.nodelabels.NodeLabel; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeLabelsUpdateSchedulerEvent; import org.apache.hadoop.yarn.util.resource.Resources; @@ -363,8 +360,8 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager { // no label in the past if (oldLabels.isEmpty()) { // update labels - NodeLabel label = labelCollections.get(NO_LABEL); - label.removeNode(oldNM.resource); + Label label = labelCollections.get(NO_LABEL); + Resources.subtractFrom(label.getResource(), oldNM.resource); // update queues, all queue can access this node for (Queue q : queueCollections.values()) { @@ -373,11 +370,11 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager { } else { // update labels for (String labelName : oldLabels) { - NodeLabel label = labelCollections.get(labelName); + Label label = labelCollections.get(labelName); if (null == label) { continue; } - label.removeNode(oldNM.resource); + Resources.subtractFrom(label.getResource(), oldNM.resource); } // update queues, only queue can access this node will be subtract @@ -398,8 +395,8 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager { // no label in the past if (newLabels.isEmpty()) { // update labels - NodeLabel label = labelCollections.get(NO_LABEL); - label.addNode(newNM.resource); + Label label = labelCollections.get(NO_LABEL); + Resources.addTo(label.getResource(), newNM.resource); // update queues, all queue can access this node for (Queue q : queueCollections.values()) { @@ -408,8 +405,8 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager { } else { // update labels for (String labelName : newLabels) { - NodeLabel label = labelCollections.get(labelName); - label.addNode(newNM.resource); + Label label = labelCollections.get(labelName); + Resources.addTo(label.getResource(), newNM.resource); } // update queues, only queue can access this node will be subtract @@ -478,21 +475,4 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager { public void setRMContext(RMContext rmContext) { this.rmContext = rmContext; } - - public List pullRMNodeLabelsInfo() { - try { - readLock.lock(); - List infos = new ArrayList(); - - for (Entry entry : labelCollections.entrySet()) { - NodeLabel label = entry.getValue(); - infos.add(label.getCopy()); - } - - Collections.sort(infos); - return infos; - } finally { - readLock.unlock(); - } - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java index db00bb0492..ce8fd9e226 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java @@ -33,8 +33,7 @@ public class NavBlock extends HtmlBlock { h3("Cluster"). ul(). li().a(url("cluster"), "About")._(). - li().a(url("nodes"), "Nodes")._(). - li().a(url("nodelabels"), "Node Labels")._(); + li().a(url("nodes"), "Nodes")._(); UL>>> subAppsList = mainList. li().a(url("apps"), "Applications"). ul(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeLabelsPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeLabelsPage.java deleted file mode 100644 index 5e8c1ed5bf..0000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeLabelsPage.java +++ /dev/null @@ -1,91 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.resourcemanager.webapp; - -import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID; - -import org.apache.hadoop.yarn.nodelabels.NodeLabel; -import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; -import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; -import org.apache.hadoop.yarn.webapp.SubView; -import org.apache.hadoop.yarn.webapp.YarnWebParams; -import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; -import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; -import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY; -import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TR; -import org.apache.hadoop.yarn.webapp.view.HtmlBlock; - -import com.google.inject.Inject; - -public class NodeLabelsPage extends RmView { - static class NodeLabelsBlock extends HtmlBlock { - final ResourceManager rm; - - @Inject - NodeLabelsBlock(ResourceManager rm, ViewContext ctx) { - super(ctx); - this.rm = rm; - } - - @Override - protected void render(Block html) { - TBODY> tbody = html.table("#nodelabels"). - thead(). - tr(). - th(".name", "Label Name"). - th(".numOfActiveNMs", "Num Of Active NMs"). - th(".totalResource", "Total Resource"). - _()._(). - tbody(); - - RMNodeLabelsManager nlm = rm.getRMContext().getNodeLabelManager(); - for (NodeLabel info : nlm.pullRMNodeLabelsInfo()) { - TR>> row = - tbody.tr().td( - info.getLabelName().isEmpty() ? "" : info - .getLabelName()); - int nActiveNMs = info.getNumActiveNMs(); - if (nActiveNMs > 0) { - row = row.td() - .a(url("nodes", - "?" + YarnWebParams.NODE_LABEL + "=" + info.getLabelName()), - String.valueOf(nActiveNMs)) - ._(); - } else { - row = row.td(String.valueOf(nActiveNMs)); - } - row.td(info.getResource().toString())._(); - } - tbody._()._(); - } - } - - @Override protected void preHead(Page.HTML<_> html) { - commonPreHead(html); - String title = "Node labels of the cluster"; - setTitle(title); - set(DATATABLES_ID, "nodelabels"); - setTableStyles(html, "nodelabels", ".healthStatus {width:10em}", - ".healthReport {width:10em}"); - } - - @Override protected Class content() { - return NodeLabelsBlock.class; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java index f28a9a88bc..d3849ae5fc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java @@ -1,25 +1,24 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ package org.apache.hadoop.yarn.server.resourcemanager.webapp; -import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_STATE; -import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_LABEL; +import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebApp.NODE_STATE; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID; @@ -29,9 +28,7 @@ import java.util.Collection; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.NodeState; -import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; -import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; @@ -63,20 +60,26 @@ class NodesPage extends RmView { ResourceScheduler sched = rm.getResourceScheduler(); String type = $(NODE_STATE); - String labelFilter = $(NODE_LABEL, CommonNodeLabelsManager.ANY).trim(); - TBODY> tbody = - html.table("#nodes").thead().tr().th(".nodelabels", "Node Labels") - .th(".rack", "Rack").th(".state", "Node State") - .th(".nodeaddress", "Node Address") - .th(".nodehttpaddress", "Node HTTP Address") - .th(".lastHealthUpdate", "Last health-update") - .th(".healthReport", "Health-report") - .th(".containers", "Containers").th(".mem", "Mem Used") - .th(".mem", "Mem Avail").th(".vcores", "VCores Used") - .th(".vcores", "VCores Avail") - .th(".nodeManagerVersion", "Version")._()._().tbody(); + TBODY> tbody = html.table("#nodes"). + thead(). + tr(). + th(".nodelabels", "Node Labels"). + th(".rack", "Rack"). + th(".state", "Node State"). + th(".nodeaddress", "Node Address"). + th(".nodehttpaddress", "Node HTTP Address"). + th(".lastHealthUpdate", "Last health-update"). + th(".healthReport", "Health-report"). + th(".containers", "Containers"). + th(".mem", "Mem Used"). + th(".mem", "Mem Avail"). + th(".vcores", "VCores Used"). + th(".vcores", "VCores Avail"). + th(".nodeManagerVersion", "Version"). + _()._(). + tbody(); NodeState stateFilter = null; - if (type != null && !type.isEmpty()) { + if(type != null && !type.isEmpty()) { stateFilter = NodeState.valueOf(type.toUpperCase()); } Collection rmNodes = this.rm.getRMContext().getRMNodes().values(); @@ -94,9 +97,9 @@ class NodesPage extends RmView { } } for (RMNode ni : rmNodes) { - if (stateFilter != null) { + if(stateFilter != null) { NodeState state = ni.getState(); - if (!stateFilter.equals(state)) { + if(!stateFilter.equals(state)) { continue; } } else { @@ -106,71 +109,61 @@ class NodesPage extends RmView { continue; } } - // Besides state, we need to filter label as well. - if (!labelFilter.equals(RMNodeLabelsManager.ANY)) { - if (labelFilter.isEmpty()) { - // Empty label filter means only shows nodes without label - if (!ni.getNodeLabels().isEmpty()) { - continue; - } - } else if (!ni.getNodeLabels().contains(labelFilter)) { - // Only nodes have given label can show on web page. - continue; - } - } NodeInfo info = new NodeInfo(ni, sched); - int usedMemory = (int) info.getUsedMemory(); - int availableMemory = (int) info.getAvailableMemory(); - TR>> row = - tbody.tr().td(StringUtils.join(",", info.getNodeLabels())) - .td(info.getRack()).td(info.getState()).td(info.getNodeId()); + int usedMemory = (int)info.getUsedMemory(); + int availableMemory = (int)info.getAvailableMemory(); + TR>> row = tbody.tr(). + td(StringUtils.join(",", info.getNodeLabels())). + td(info.getRack()). + td(info.getState()). + td(info.getNodeId()); if (isInactive) { row.td()._("N/A")._(); } else { String httpAddress = info.getNodeHTTPAddress(); - row.td().a("//" + httpAddress, httpAddress)._(); + row.td().a("//" + httpAddress, + httpAddress)._(); } - row.td().br().$title(String.valueOf(info.getLastHealthUpdate()))._() - ._(Times.format(info.getLastHealthUpdate()))._() - .td(info.getHealthReport()) - .td(String.valueOf(info.getNumContainers())).td().br() - .$title(String.valueOf(usedMemory))._() - ._(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))._().td().br() - .$title(String.valueOf(availableMemory))._() - ._(StringUtils.byteDesc(availableMemory * BYTES_IN_MB))._() - .td(String.valueOf(info.getUsedVirtualCores())) - .td(String.valueOf(info.getAvailableVirtualCores())) - .td(ni.getNodeManagerVersion())._(); + row.td().br().$title(String.valueOf(info.getLastHealthUpdate()))._(). + _(Times.format(info.getLastHealthUpdate()))._(). + td(info.getHealthReport()). + td(String.valueOf(info.getNumContainers())). + td().br().$title(String.valueOf(usedMemory))._(). + _(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))._(). + td().br().$title(String.valueOf(availableMemory))._(). + _(StringUtils.byteDesc(availableMemory * BYTES_IN_MB))._(). + td(String.valueOf(info.getUsedVirtualCores())). + td(String.valueOf(info.getAvailableVirtualCores())). + td(ni.getNodeManagerVersion()). + _(); } tbody._()._(); } } - @Override - protected void preHead(Page.HTML<_> html) { + @Override protected void preHead(Page.HTML<_> html) { commonPreHead(html); String type = $(NODE_STATE); String title = "Nodes of the cluster"; - if (type != null && !type.isEmpty()) { - title = title + " (" + type + ")"; + if(type != null && !type.isEmpty()) { + title = title+" ("+type+")"; } setTitle(title); set(DATATABLES_ID, "nodes"); set(initID(DATATABLES, "nodes"), nodesTableInit()); setTableStyles(html, "nodes", ".healthStatus {width:10em}", - ".healthReport {width:10em}"); + ".healthReport {width:10em}"); } - @Override - protected Class content() { + @Override protected Class content() { return NodesBlock.class; } private String nodesTableInit() { StringBuilder b = tableInit().append(", aoColumnDefs: ["); b.append("{'bSearchable': false, 'aTargets': [ 6 ]}"); - b.append(", {'sType': 'title-numeric', 'bSearchable': false, " - + "'aTargets': [ 7, 8 ] }"); + b.append(", {'sType': 'title-numeric', 'bSearchable': false, " + + "'aTargets': [ 7, 8 ] }"); b.append(", {'sType': 'title-numeric', 'aTargets': [ 4 ]}"); b.append("]}"); return b.toString(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java index c0e68349e4..67c73b8127 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java @@ -61,7 +61,6 @@ public class RMWebApp extends WebApp implements YarnWebParams { route(pajoin("/app", APPLICATION_ID), RmController.class, "app"); route("/scheduler", RmController.class, "scheduler"); route(pajoin("/queue", QUEUE_NAME), RmController.class, "queue"); - route("/nodelabels", RmController.class, "nodelabels"); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java index 972432b988..f186bf498b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java @@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.util.StringHelper; import org.apache.hadoop.yarn.webapp.Controller; +import org.apache.hadoop.yarn.webapp.WebAppException; import org.apache.hadoop.yarn.webapp.YarnWebParams; import com.google.inject.Inject; @@ -92,9 +93,4 @@ public class RmController extends Controller { public void submit() { setTitle("Application Submission Not Allowed"); } - - public void nodelabels() { - setTitle("Node Labels"); - render(NodeLabelsPage.class); - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java index 2d863d1af5..278c151ff6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java @@ -30,13 +30,11 @@ import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; -import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo; -import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; /** @@ -55,12 +53,7 @@ public class MockNodes { // One unhealthy node per rack. list.add(nodeInfo(i, perNode, NodeState.UNHEALTHY)); } - if (j == 0) { - // One node with label - list.add(nodeInfo(i, perNode, NodeState.RUNNING, ImmutableSet.of("x"))); - } else { - list.add(newNodeInfo(i, perNode)); - } + list.add(newNodeInfo(i, perNode)); } } return list; @@ -107,12 +100,10 @@ public class MockNodes { private String healthReport; private long lastHealthReportTime; private NodeState state; - private Set labels; public MockRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress, Resource perNode, String rackName, String healthReport, - long lastHealthReportTime, int cmdPort, String hostName, NodeState state, - Set labels) { + long lastHealthReportTime, int cmdPort, String hostName, NodeState state) { this.nodeId = nodeId; this.nodeAddr = nodeAddr; this.httpAddress = httpAddress; @@ -123,7 +114,6 @@ public class MockNodes { this.cmdPort = cmdPort; this.hostName = hostName; this.state = state; - this.labels = labels; } @Override @@ -217,33 +207,16 @@ public class MockNodes { @Override public Set getNodeLabels() { - if (labels != null) { - return labels; - } - return CommonNodeLabelsManager.EMPTY_STRING_SET; + return RMNodeLabelsManager.EMPTY_STRING_SET; } }; - private static RMNode buildRMNode(int rack, final Resource perNode, - NodeState state, String httpAddr) { - return buildRMNode(rack, perNode, state, httpAddr, null); - } - - private static RMNode buildRMNode(int rack, final Resource perNode, - NodeState state, String httpAddr, Set labels) { - return buildRMNode(rack, perNode, state, httpAddr, NODE_ID++, null, 123, - labels); - } - - private static RMNode buildRMNode(int rack, final Resource perNode, - NodeState state, String httpAddr, int hostnum, String hostName, int port) { - return buildRMNode(rack, perNode, state, httpAddr, hostnum, hostName, port, - null); + private static RMNode buildRMNode(int rack, final Resource perNode, NodeState state, String httpAddr) { + return buildRMNode(rack, perNode, state, httpAddr, NODE_ID++, null, 123); } private static RMNode buildRMNode(int rack, final Resource perNode, - NodeState state, String httpAddr, int hostnum, String hostName, int port, - Set labels) { + NodeState state, String httpAddr, int hostnum, String hostName, int port) { final String rackName = "rack"+ rack; final int nid = hostnum; final String nodeAddr = hostName + ":" + nid; @@ -255,18 +228,13 @@ public class MockNodes { final String httpAddress = httpAddr; String healthReport = (state == NodeState.UNHEALTHY) ? null : "HealthyMe"; return new MockRMNodeImpl(nodeID, nodeAddr, httpAddress, perNode, - rackName, healthReport, 0, nid, hostName, state, labels); + rackName, healthReport, 0, nid, hostName, state); } public static RMNode nodeInfo(int rack, final Resource perNode, NodeState state) { return buildRMNode(rack, perNode, state, "N/A"); } - - public static RMNode nodeInfo(int rack, final Resource perNode, - NodeState state, Set labels) { - return buildRMNode(rack, perNode, state, "N/A", labels); - } public static RMNode newNodeInfo(int rack, final Resource perNode) { return buildRMNode(rack, perNode, NodeState.RUNNING, "localhost:0"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java index e21fcf9d6a..842eaecad3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java @@ -839,7 +839,7 @@ public class TestWorkPreservingRMRestart { // Test if RM on recovery receives the container release request from AM // before it receives the container status reported by NM for recovery. this // container should not be recovered. - @Test (timeout = 50000) + @Test (timeout = 30000) public void testReleasedContainerNotRecovered() throws Exception { MemoryRMStateStore memStore = new MemoryRMStateStore(); memStore.init(conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java index e4cdc71c78..ed675f3736 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java @@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.nodelabels; import java.io.IOException; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Set; @@ -28,7 +27,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; -import org.apache.hadoop.yarn.nodelabels.NodeLabel; import org.apache.hadoop.yarn.nodelabels.NodeLabelTestBase; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; @@ -430,35 +428,4 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase { Assert.fail("IOException from removeLabelsFromNode " + e); } } - - private void checkNodeLabelInfo(List infos, String labelName, int activeNMs, int memory) { - for (NodeLabel info : infos) { - if (info.getLabelName().equals(labelName)) { - Assert.assertEquals(activeNMs, info.getNumActiveNMs()); - Assert.assertEquals(memory, info.getResource().getMemory()); - return; - } - } - Assert.fail("Failed to find info has label=" + labelName); - } - - @Test(timeout = 5000) - public void testPullRMNodeLabelsInfo() throws IOException { - mgr.addToCluserNodeLabels(toSet("x", "y", "z")); - mgr.activateNode(NodeId.newInstance("n1", 1), Resource.newInstance(10, 0)); - mgr.activateNode(NodeId.newInstance("n2", 1), Resource.newInstance(10, 0)); - mgr.activateNode(NodeId.newInstance("n3", 1), Resource.newInstance(10, 0)); - mgr.activateNode(NodeId.newInstance("n4", 1), Resource.newInstance(10, 0)); - mgr.activateNode(NodeId.newInstance("n5", 1), Resource.newInstance(10, 0)); - mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("x"), - toNodeId("n2"), toSet("x"), toNodeId("n3"), toSet("y"))); - - // x, y, z and "" - List infos = mgr.pullRMNodeLabelsInfo(); - Assert.assertEquals(4, infos.size()); - checkNodeLabelInfo(infos, RMNodeLabelsManager.NO_LABEL, 2, 20); - checkNodeLabelInfo(infos, "x", 2, 20); - checkNodeLabelInfo(infos, "y", 1, 10); - checkNodeLabelInfo(infos, "z", 0, 0); - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java index 62713cfc7c..bb38079cce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java @@ -106,49 +106,4 @@ public class TestNodesPage { * numberOfActualTableHeaders + numberOfThInMetricsTable)).print( "