From 42b32fbbdc93259480d3eb7cea5ce22a5193fa68 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Sun, 8 Oct 2023 08:20:11 +0800 Subject: [PATCH] YARN-11583. Improve Node Link for YARN Federation Web Page. (#6145) Contributed by Shilun Fan. Reviewed-by: Inigo Goiri Signed-off-by: Shilun Fan --- .../router/webapp/MetricsOverviewTable.java | 20 ++++++++---- .../server/router/webapp/NodeLabelsBlock.java | 9 ++++++ .../server/router/webapp/NodeLabelsPage.java | 11 +++++-- .../yarn/server/router/webapp/NodesBlock.java | 32 ++++++++++++++++--- .../yarn/server/router/webapp/NodesPage.java | 9 ++++-- .../server/router/webapp/RouterBlock.java | 2 ++ .../router/webapp/RouterWebServiceUtil.java | 9 ++++++ .../router/webapp/TestRouterWebServices.java | 2 +- 8 files changed, 78 insertions(+), 16 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/MetricsOverviewTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/MetricsOverviewTable.java index f27692b5ad..aeb953915a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/MetricsOverviewTable.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/MetricsOverviewTable.java @@ -160,13 +160,19 @@ private void initFederationClusterNodesMetrics(Hamlet.DIV div, // Initialize table data information tbody().$class("ui-widget-content"). tr(). - td(String.valueOf(metrics.getActiveNodes())). - td(String.valueOf(metrics.getDecommissioningNodes())). - td(String.valueOf(metrics.getDecommissionedNodes())). - td(String.valueOf(metrics.getLostNodes())). - td(String.valueOf(metrics.getUnhealthyNodes())). - td(String.valueOf(metrics.getRebootedNodes())). - td(String.valueOf(metrics.getShutdownNodes())). + td().a(url("nodes"), String.valueOf(metrics.getActiveNodes())).__(). + td().a(url("nodes/router/?node.state=decommissioning"), + String.valueOf(metrics.getDecommissioningNodes())).__(). + td().a(url("nodes/router/?node.state=decommissioned"), + String.valueOf(metrics.getDecommissionedNodes())).__(). + td().a(url("nodes/router/?node.state=lost"), + String.valueOf(metrics.getLostNodes())).__(). + td().a(url("nodes/router/?node.state=unhealthy"), + String.valueOf(metrics.getUnhealthyNodes())).__(). + td().a(url("nodes/router/?node.state=rebooted"), + String.valueOf(metrics.getRebootedNodes())).__(). + td().a(url("nodes/router/?node.state=shutdown"), + String.valueOf(metrics.getShutdownNodes())).__(). __(). __().__(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsBlock.java index 37d1d3c95a..2ee813934f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsBlock.java @@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.PartitionInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; import org.apache.hadoop.yarn.server.router.Router; +import org.apache.hadoop.yarn.webapp.YarnWebParams; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @@ -158,6 +159,14 @@ private void initYarnFederationNodeLabelsOfCluster(NodeLabelsInfo nodeLabelsInfo String type = (info.getExclusivity()) ? "Exclusive Partition" : "Non Exclusive Partition"; row = row.td(type); int nActiveNMs = info.getActiveNMs(); + if (nActiveNMs > 0) { + row = row.td().a(url("nodes", + "?" + YarnWebParams.NODE_LABEL + "=" + info.getName()), String.valueOf(nActiveNMs)) + .__(); + } else { + row = row.td(String.valueOf(nActiveNMs)); + } + row = row.td(String.valueOf(nActiveNMs)); PartitionInfo partitionInfo = info.getPartitionInfo(); ResourceInfo available = partitionInfo.getResourceAvailable(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsPage.java index 9b3cea4681..5ab462ee0f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsPage.java @@ -21,7 +21,9 @@ import org.apache.hadoop.yarn.webapp.SubView; import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet; +import static org.apache.hadoop.yarn.server.router.webapp.RouterWebServiceUtil.generateWebTitle; import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_SC; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_LABEL; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID; /** @@ -33,10 +35,15 @@ public class NodeLabelsPage extends RouterView { protected void preHead(Hamlet.HTML<__> html) { commonPreHead(html); String type = $(NODE_SC); + String nodeLabel = $(NODE_LABEL); String title = "Node labels of the cluster"; - if (type != null && !type.isEmpty()) { - title = title + " (" + type + ")"; + + if (nodeLabel != null && !nodeLabel.isEmpty()) { + title = generateWebTitle(title, nodeLabel); + } else if (type != null && !type.isEmpty()) { + title = generateWebTitle(title, type); } + setTitle(title); set(DATATABLES_ID, "nodelabels"); setTableStyles(html, "nodelabels", ".healthStatus {width:10em}", ".healthReport {width:10em}"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java index f5c0f9f9d1..4544ed4ed4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java @@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo; @@ -40,6 +41,8 @@ import java.util.Date; import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_SC; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_LABEL; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_STATE; /** * Nodes block for the Router Web UI. @@ -61,12 +64,15 @@ protected void render(Block html) { // Get subClusterName String subClusterName = $(NODE_SC); + String state = $(NODE_STATE); + String nodeLabel = $(NODE_LABEL); // We will try to get the subClusterName. // If the subClusterName is not empty, // it means that we need to get the Node list of a subCluster. NodesInfo nodesInfo; - if (subClusterName != null && !subClusterName.isEmpty()) { + if (subClusterName != null && !subClusterName.isEmpty() && + !ROUTER.equalsIgnoreCase(subClusterName)) { initSubClusterMetricsOverviewTable(html, subClusterName); nodesInfo = getSubClusterNodesInfo(subClusterName); } else { @@ -76,7 +82,7 @@ protected void render(Block html) { } // Initialize NodeInfo List - initYarnFederationNodesOfCluster(nodesInfo, html); + initYarnFederationNodesOfCluster(nodesInfo, html, state, nodeLabel); } private NodesInfo getYarnFederationNodesInfo(boolean isEnabled) { @@ -100,7 +106,7 @@ private NodesInfo getSubClusterNodesInfo(String subCluster) { if (subClusterInfo != null) { // Prepare webAddress String webAddress = subClusterInfo.getRMWebServiceAddress(); - String herfWebAppAddress = ""; + String herfWebAppAddress; if (webAddress != null && !webAddress.isEmpty()) { herfWebAppAddress = WebAppUtils.getHttpSchemePrefix(this.router.getConfig()) + webAddress; @@ -124,7 +130,8 @@ private NodesInfo getSubClusterNodesInfoByWebAddress(String webAddress) { return nodes; } - private void initYarnFederationNodesOfCluster(NodesInfo nodesInfo, Block html) { + private void initYarnFederationNodesOfCluster(NodesInfo nodesInfo, Block html, + String filterState, String filterLabel) { TBODY> tbody = html.table("#nodes").thead().tr() .th(".nodelabels", "Node Labels") .th(".rack", "Rack") @@ -143,6 +150,23 @@ private void initYarnFederationNodesOfCluster(NodesInfo nodesInfo, Block html) { if (nodesInfo != null && CollectionUtils.isNotEmpty(nodesInfo.getNodes())) { for (NodeInfo info : nodesInfo.getNodes()) { + if (filterState != null && !filterState.isEmpty() && !filterState.equals(info.getState())) { + continue; + } + + // Besides state, we need to filter label as well. + if (!filterLabel.equals(RMNodeLabelsManager.ANY)) { + if (filterLabel.isEmpty()) { + // Empty label filter means only shows nodes without label + if (!info.getNodeLabels().isEmpty()) { + continue; + } + } else if (!info.getNodeLabels().contains(filterLabel)) { + // Only nodes have given label can show on web page. + continue; + } + } + int usedMemory = (int) info.getUsedMemory(); int availableMemory = (int) info.getAvailableMemory(); TR>> row = tbody.tr(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesPage.java index 0723cff792..386e540b7d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesPage.java @@ -18,7 +18,9 @@ package org.apache.hadoop.yarn.server.router.webapp; +import static org.apache.hadoop.yarn.server.router.webapp.RouterWebServiceUtil.generateWebTitle; import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_SC; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_STATE; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID; @@ -32,9 +34,12 @@ class NodesPage extends RouterView { protected void preHead(Page.HTML<__> html) { commonPreHead(html); String type = $(NODE_SC); + String state = $(NODE_STATE); String title = "Nodes of the cluster"; - if (type != null && !type.isEmpty()) { - title = title + " (" + type + ")"; + if (state != null && !state.isEmpty()) { + title = generateWebTitle(title, state); + } else if (type != null && !type.isEmpty()) { + title = generateWebTitle(title, type); } setTitle(title); set(DATATABLES_ID, "nodes"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterBlock.java index c811c4d48c..55bcb81f25 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterBlock.java @@ -55,6 +55,8 @@ public abstract class RouterBlock extends HtmlBlock { private final FederationStateStoreFacade facade; private final Configuration conf; + public static final String ROUTER = "router"; + public RouterBlock(Router router, ViewContext ctx) { super(ctx); this.ctx = ctx; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java index 02913a43cb..f2c385fd02 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java @@ -764,4 +764,13 @@ public static UserGroupInformation getKerberosUserGroupInformation(Configuration // return caller UGI return callerUGI; } + + public static String generateWebTitle(String title, String msg) { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append(title); + stringBuilder.append(" ("); + stringBuilder.append(msg); + stringBuilder.append(")"); + return stringBuilder.toString(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServices.java index 5bf8db04b0..dbcd7db21c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServices.java @@ -334,7 +334,7 @@ public RESTRequestInterceptor run() throws Exception { Assert.assertNotNull(client1.interceptor); Assert.assertNotNull(client2.interceptor); - Assert.assertTrue(client1.interceptor == client2.interceptor); + Assert.assertSame(client1.interceptor, client2.interceptor); } }