YARN-11327. [Federation] Refactoring Yarn Router's Node Web Page. (#5009)
This commit is contained in:
parent
bfce21ee08
commit
647457e6ab
@ -39,6 +39,7 @@ public interface YarnWebParams {
|
|||||||
String QUEUE_NAME = "queue.name";
|
String QUEUE_NAME = "queue.name";
|
||||||
String NODE_STATE = "node.state";
|
String NODE_STATE = "node.state";
|
||||||
String NODE_LABEL = "node.label";
|
String NODE_LABEL = "node.label";
|
||||||
|
String NODE_SC = "node.subcluster";
|
||||||
String WEB_UI_TYPE = "web.ui.type";
|
String WEB_UI_TYPE = "web.ui.type";
|
||||||
String NEXT_REFRESH_INTERVAL = "next.refresh.interval";
|
String NEXT_REFRESH_INTERVAL = "next.refresh.interval";
|
||||||
String ERROR_MESSAGE = "error.message";
|
String ERROR_MESSAGE = "error.message";
|
||||||
|
@ -68,6 +68,7 @@ public class NodeInfo {
|
|||||||
protected ResourceInfo availableResource;
|
protected ResourceInfo availableResource;
|
||||||
protected NodeAttributesInfo nodeAttributesInfo;
|
protected NodeAttributesInfo nodeAttributesInfo;
|
||||||
private ResourceInfo totalResource;
|
private ResourceInfo totalResource;
|
||||||
|
private String subClusterId;
|
||||||
|
|
||||||
public NodeInfo() {
|
public NodeInfo() {
|
||||||
} // JAXB needs this
|
} // JAXB needs this
|
||||||
@ -287,4 +288,12 @@ public void setTotalResource(ResourceInfo total) {
|
|||||||
public ResourceInfo getTotalResource() {
|
public ResourceInfo getTotalResource() {
|
||||||
return this.totalResource;
|
return this.totalResource;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public String getSubClusterId() {
|
||||||
|
return subClusterId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setSubClusterId(String subClusterId) {
|
||||||
|
this.subClusterId = subClusterId;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -18,6 +18,7 @@
|
|||||||
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
|
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
import java.util.Collection;
|
||||||
|
|
||||||
import javax.xml.bind.annotation.XmlAccessType;
|
import javax.xml.bind.annotation.XmlAccessType;
|
||||||
import javax.xml.bind.annotation.XmlAccessorType;
|
import javax.xml.bind.annotation.XmlAccessorType;
|
||||||
@ -43,4 +44,8 @@ public ArrayList<NodeInfo> getNodes() {
|
|||||||
public void addAll(ArrayList<NodeInfo> nodesInfo) {
|
public void addAll(ArrayList<NodeInfo> nodesInfo) {
|
||||||
node.addAll(nodesInfo);
|
node.addAll(nodesInfo);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void addAll(Collection<NodeInfo> nodesInfo) {
|
||||||
|
node.addAll(nodesInfo);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -112,9 +112,6 @@ private void initFederationSubClusterDetailTableJs(Block html,
|
|||||||
private void initHtmlPageFederation(Block html, boolean isEnabled) {
|
private void initHtmlPageFederation(Block html, boolean isEnabled) {
|
||||||
List<Map<String, String>> lists = new ArrayList<>();
|
List<Map<String, String>> lists = new ArrayList<>();
|
||||||
|
|
||||||
// If Yarn Federation is not enabled, the user needs to be prompted.
|
|
||||||
initUserHelpInformationDiv(html, isEnabled);
|
|
||||||
|
|
||||||
// Table header
|
// Table header
|
||||||
TBODY<TABLE<Hamlet>> tbody =
|
TBODY<TABLE<Hamlet>> tbody =
|
||||||
html.table("#rms").$class("cell-border").$style("width:100%").thead().tr()
|
html.table("#rms").$class("cell-border").$style("width:100%").thead().tr()
|
||||||
|
@ -32,6 +32,7 @@
|
|||||||
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.util.Collection;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
public class MetricsOverviewTable extends RouterBlock {
|
public class MetricsOverviewTable extends RouterBlock {
|
||||||
@ -58,7 +59,31 @@ protected void render(Block html) {
|
|||||||
try {
|
try {
|
||||||
initFederationClusterAppsMetrics(div, routerClusterMetrics);
|
initFederationClusterAppsMetrics(div, routerClusterMetrics);
|
||||||
initFederationClusterNodesMetrics(div, routerClusterMetrics);
|
initFederationClusterNodesMetrics(div, routerClusterMetrics);
|
||||||
initFederationClusterSchedulersMetrics(div, routerClusterMetrics);
|
List<SubClusterInfo> subClusters = getSubClusterInfoList();
|
||||||
|
initFederationClusterSchedulersMetrics(div, routerClusterMetrics, subClusters);
|
||||||
|
} catch (Exception e) {
|
||||||
|
LOG.error("MetricsOverviewTable init error.", e);
|
||||||
|
}
|
||||||
|
div.__();
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void render(Block html, String subClusterId) {
|
||||||
|
// Initialize page styles
|
||||||
|
html.style(".metrics {margin-bottom:5px}");
|
||||||
|
|
||||||
|
// get subClusterId ClusterMetrics Info
|
||||||
|
ClusterMetricsInfo clusterMetricsInfo =
|
||||||
|
getClusterMetricsInfoBySubClusterId(subClusterId);
|
||||||
|
RouterClusterMetrics routerClusterMetrics =
|
||||||
|
new RouterClusterMetrics(clusterMetricsInfo, subClusterId);
|
||||||
|
|
||||||
|
// metrics div
|
||||||
|
Hamlet.DIV<Hamlet> div = html.div().$class("metrics");
|
||||||
|
try {
|
||||||
|
initFederationClusterAppsMetrics(div, routerClusterMetrics);
|
||||||
|
initFederationClusterNodesMetrics(div, routerClusterMetrics);
|
||||||
|
Collection<SubClusterInfo> subClusters = getSubClusterInfoList(subClusterId);
|
||||||
|
initFederationClusterSchedulersMetrics(div, routerClusterMetrics, subClusters);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
LOG.error("MetricsOverviewTable init error.", e);
|
LOG.error("MetricsOverviewTable init error.", e);
|
||||||
}
|
}
|
||||||
@ -74,7 +99,7 @@ protected void render(Block html) {
|
|||||||
*/
|
*/
|
||||||
private void initFederationClusterAppsMetrics(Hamlet.DIV<Hamlet> div,
|
private void initFederationClusterAppsMetrics(Hamlet.DIV<Hamlet> div,
|
||||||
RouterClusterMetrics metrics) {
|
RouterClusterMetrics metrics) {
|
||||||
div.h3("Federation Cluster Metrics").
|
div.h3(metrics.getWebPageTitlePrefix() + " Cluster Metrics").
|
||||||
table("#metricsoverview").
|
table("#metricsoverview").
|
||||||
thead().$class("ui-widget-header").
|
thead().$class("ui-widget-header").
|
||||||
// Initialize table header information
|
// Initialize table header information
|
||||||
@ -116,7 +141,7 @@ private void initFederationClusterAppsMetrics(Hamlet.DIV<Hamlet> div,
|
|||||||
*/
|
*/
|
||||||
private void initFederationClusterNodesMetrics(Hamlet.DIV<Hamlet> div,
|
private void initFederationClusterNodesMetrics(Hamlet.DIV<Hamlet> div,
|
||||||
RouterClusterMetrics metrics) {
|
RouterClusterMetrics metrics) {
|
||||||
div.h3("Federation Cluster Nodes Metrics").
|
div.h3(metrics.getWebPageTitlePrefix() + " Cluster Nodes Metrics").
|
||||||
table("#nodemetricsoverview").
|
table("#nodemetricsoverview").
|
||||||
thead().$class("ui-widget-header").
|
thead().$class("ui-widget-header").
|
||||||
// Initialize table header information
|
// Initialize table header information
|
||||||
@ -149,17 +174,17 @@ private void initFederationClusterNodesMetrics(Hamlet.DIV<Hamlet> div,
|
|||||||
*
|
*
|
||||||
* @param div data display div.
|
* @param div data display div.
|
||||||
* @param metrics data metric information.
|
* @param metrics data metric information.
|
||||||
|
* @param subclusters active subcluster List.
|
||||||
* @throws YarnException yarn error.
|
* @throws YarnException yarn error.
|
||||||
* @throws IOException io error.
|
* @throws IOException io error.
|
||||||
* @throws InterruptedException interrupt error.
|
* @throws InterruptedException interrupt error.
|
||||||
*/
|
*/
|
||||||
private void initFederationClusterSchedulersMetrics(Hamlet.DIV<Hamlet> div,
|
private void initFederationClusterSchedulersMetrics(Hamlet.DIV<Hamlet> div,
|
||||||
RouterClusterMetrics metrics) throws YarnException, IOException, InterruptedException {
|
RouterClusterMetrics metrics, Collection<SubClusterInfo> subclusters)
|
||||||
// Sort the SubClusters.
|
throws YarnException, IOException, InterruptedException {
|
||||||
List<SubClusterInfo> subclusters = getSubClusterInfoList();
|
|
||||||
|
|
||||||
Hamlet.TBODY<Hamlet.TABLE<Hamlet.DIV<Hamlet>>> fsMetricsScheduleTr =
|
Hamlet.TBODY<Hamlet.TABLE<Hamlet.DIV<Hamlet>>> fsMetricsScheduleTr =
|
||||||
div.h3("Federation Scheduler Metrics").
|
div.h3(metrics.getWebPageTitlePrefix() + " Scheduler Metrics").
|
||||||
table("#schedulermetricsoverview").
|
table("#schedulermetricsoverview").
|
||||||
thead().$class("ui-widget-header").
|
thead().$class("ui-widget-header").
|
||||||
tr().
|
tr().
|
||||||
@ -202,7 +227,7 @@ private void initFederationClusterSchedulersMetrics(Hamlet.DIV<Hamlet> div,
|
|||||||
|
|
||||||
private void initSubClusterOverViewTable(RouterClusterMetrics metrics,
|
private void initSubClusterOverViewTable(RouterClusterMetrics metrics,
|
||||||
Hamlet.TBODY<Hamlet.TABLE<Hamlet.DIV<Hamlet>>> fsMetricsScheduleTr,
|
Hamlet.TBODY<Hamlet.TABLE<Hamlet.DIV<Hamlet>>> fsMetricsScheduleTr,
|
||||||
List<SubClusterInfo> subclusters) {
|
Collection<SubClusterInfo> subclusters) {
|
||||||
|
|
||||||
// configuration
|
// configuration
|
||||||
Configuration config = this.router.getConfig();
|
Configuration config = this.router.getConfig();
|
||||||
|
@ -18,29 +18,59 @@
|
|||||||
|
|
||||||
package org.apache.hadoop.yarn.server.router.webapp;
|
package org.apache.hadoop.yarn.server.router.webapp;
|
||||||
|
|
||||||
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
|
import com.google.inject.Inject;
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.yarn.server.router.Router;
|
||||||
|
import org.apache.hadoop.yarn.server.webapp.WebPageUtils;
|
||||||
|
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Navigation block for the Router Web UI.
|
* Navigation block for the Router Web UI.
|
||||||
*/
|
*/
|
||||||
public class NavBlock extends HtmlBlock {
|
public class NavBlock extends RouterBlock {
|
||||||
|
|
||||||
|
private Router router;
|
||||||
|
|
||||||
|
@Inject
|
||||||
|
public NavBlock(Router router, ViewContext ctx) {
|
||||||
|
super(router, ctx);
|
||||||
|
this.router = router;
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void render(Block html) {
|
public void render(Block html) {
|
||||||
html.
|
Hamlet.UL<Hamlet.DIV<Hamlet>> mainList = html.div("#nav").
|
||||||
div("#nav").
|
|
||||||
h3("Cluster").
|
h3("Cluster").
|
||||||
ul().
|
ul().
|
||||||
li().a(url(""), "About").__().
|
li().a(url(""), "About").__().
|
||||||
li().a(url("federation"), "Federation").__().
|
li().a(url("federation"), "Federation").__();
|
||||||
li().a(url("nodes"), "Nodes").__().
|
|
||||||
li().a(url("apps"), "Applications").__().
|
List<String> subClusterIds = getActiveSubClusterIds();
|
||||||
__().
|
|
||||||
h3("Tools").
|
Hamlet.UL<Hamlet.LI<Hamlet.UL<Hamlet.DIV<Hamlet>>>> subAppsList1 =
|
||||||
ul().
|
mainList.li().a(url("nodes"), "Nodes").ul().$style("padding:0.3em 1em 0.1em 2em");
|
||||||
li().a("/conf", "Configuration").__().
|
|
||||||
li().a("/logs", "Local logs").__().
|
// ### nodes info
|
||||||
li().a("/stacks", "Server stacks").__().
|
subAppsList1.li().__();
|
||||||
li().a("/jmx?qry=Hadoop:*", "Server metrics").__().__().__();
|
for (String subClusterId : subClusterIds) {
|
||||||
|
subAppsList1.li().a(url("nodes", subClusterId), subClusterId).__();
|
||||||
|
}
|
||||||
|
subAppsList1.__().__();
|
||||||
|
|
||||||
|
// ### applications info
|
||||||
|
mainList.li().a(url("apps"), "Applications").__();
|
||||||
|
|
||||||
|
// ### tools
|
||||||
|
Hamlet.DIV<Hamlet> sectionBefore = mainList.__();
|
||||||
|
Configuration conf = new Configuration();
|
||||||
|
Hamlet.UL<Hamlet.DIV<Hamlet>> tools = WebPageUtils.appendToolSection(sectionBefore, conf);
|
||||||
|
|
||||||
|
if (tools == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
tools.__().__();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -19,50 +19,107 @@
|
|||||||
package org.apache.hadoop.yarn.server.router.webapp;
|
package org.apache.hadoop.yarn.server.router.webapp;
|
||||||
|
|
||||||
import com.sun.jersey.api.client.Client;
|
import com.sun.jersey.api.client.Client;
|
||||||
|
import org.apache.commons.collections.CollectionUtils;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.util.StringUtils;
|
import org.apache.hadoop.util.StringUtils;
|
||||||
|
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
|
||||||
|
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
|
||||||
|
import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
|
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
|
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
|
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
|
||||||
import org.apache.hadoop.yarn.server.router.Router;
|
import org.apache.hadoop.yarn.server.router.Router;
|
||||||
import org.apache.hadoop.yarn.util.Times;
|
|
||||||
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
|
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
|
||||||
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE;
|
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE;
|
||||||
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TBODY;
|
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TBODY;
|
||||||
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TR;
|
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TR;
|
||||||
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
||||||
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
|
|
||||||
|
|
||||||
import com.google.inject.Inject;
|
import com.google.inject.Inject;
|
||||||
|
|
||||||
|
import java.util.Date;
|
||||||
|
|
||||||
|
import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_SC;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Nodes block for the Router Web UI.
|
* Nodes block for the Router Web UI.
|
||||||
*/
|
*/
|
||||||
public class NodesBlock extends HtmlBlock {
|
public class NodesBlock extends RouterBlock {
|
||||||
|
|
||||||
private static final long BYTES_IN_MB = 1024 * 1024;
|
|
||||||
|
|
||||||
private final Router router;
|
private final Router router;
|
||||||
|
|
||||||
@Inject
|
@Inject
|
||||||
NodesBlock(Router router, ViewContext ctx) {
|
NodesBlock(Router router, ViewContext ctx) {
|
||||||
super(ctx);
|
super(router, ctx);
|
||||||
this.router = router;
|
this.router = router;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void render(Block html) {
|
protected void render(Block html) {
|
||||||
// Get the node info from the federation
|
|
||||||
|
boolean isEnabled = isYarnFederationEnabled();
|
||||||
|
|
||||||
|
// Get subClusterName
|
||||||
|
String subClusterName = $(NODE_SC);
|
||||||
|
|
||||||
|
// We will try to get the subClusterName.
|
||||||
|
// If the subClusterName is not empty,
|
||||||
|
// it means that we need to get the Node list of a subCluster.
|
||||||
|
NodesInfo nodesInfo = null;
|
||||||
|
if (subClusterName != null && !subClusterName.isEmpty()) {
|
||||||
|
initSubClusterMetricsOverviewTable(html, subClusterName);
|
||||||
|
nodesInfo = getSubClusterNodesInfo(subClusterName);
|
||||||
|
} else {
|
||||||
|
// Metrics Overview Table
|
||||||
|
html.__(MetricsOverviewTable.class);
|
||||||
|
nodesInfo = getYarnFederationNodesInfo(isEnabled);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize NodeInfo List
|
||||||
|
initYarnFederationNodesOfCluster(nodesInfo, html);
|
||||||
|
}
|
||||||
|
|
||||||
|
private NodesInfo getYarnFederationNodesInfo(boolean isEnabled) {
|
||||||
|
if (isEnabled) {
|
||||||
|
String webAddress = WebAppUtils.getRouterWebAppURLWithScheme(this.router.getConfig());
|
||||||
|
return getSubClusterNodesInfoByWebAddress(webAddress);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private NodesInfo getSubClusterNodesInfo(String subCluster) {
|
||||||
|
try {
|
||||||
|
SubClusterId subClusterId = SubClusterId.newInstance(subCluster);
|
||||||
|
FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance();
|
||||||
|
SubClusterInfo subClusterInfo = facade.getSubCluster(subClusterId);
|
||||||
|
|
||||||
|
if (subClusterInfo != null) {
|
||||||
|
// Prepare webAddress
|
||||||
|
String webAddress = subClusterInfo.getRMWebServiceAddress();
|
||||||
|
String herfWebAppAddress = "";
|
||||||
|
if (webAddress != null && !webAddress.isEmpty()) {
|
||||||
|
herfWebAppAddress =
|
||||||
|
WebAppUtils.getHttpSchemePrefix(this.router.getConfig()) + webAddress;
|
||||||
|
return getSubClusterNodesInfoByWebAddress(herfWebAppAddress);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
LOG.error("get NodesInfo From SubCluster = {} error.", subCluster, e);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private NodesInfo getSubClusterNodesInfoByWebAddress(String webAddress) {
|
||||||
Configuration conf = this.router.getConfig();
|
Configuration conf = this.router.getConfig();
|
||||||
Client client = RouterWebServiceUtil.createJerseyClient(conf);
|
Client client = RouterWebServiceUtil.createJerseyClient(conf);
|
||||||
String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf);
|
|
||||||
NodesInfo nodes = RouterWebServiceUtil
|
NodesInfo nodes = RouterWebServiceUtil
|
||||||
.genericForward(webAppAddress, null, NodesInfo.class, HTTPMethods.GET,
|
.genericForward(webAddress, null, NodesInfo.class, HTTPMethods.GET,
|
||||||
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES, null, null, conf,
|
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES, null, null, conf,
|
||||||
client);
|
client);
|
||||||
|
return nodes;
|
||||||
setTitle("Nodes");
|
}
|
||||||
|
|
||||||
|
private void initYarnFederationNodesOfCluster(NodesInfo nodesInfo, Block html) {
|
||||||
TBODY<TABLE<Hamlet>> tbody = html.table("#nodes").thead().tr()
|
TBODY<TABLE<Hamlet>> tbody = html.table("#nodes").thead().tr()
|
||||||
.th(".nodelabels", "Node Labels")
|
.th(".nodelabels", "Node Labels")
|
||||||
.th(".rack", "Rack")
|
.th(".rack", "Rack")
|
||||||
@ -79,34 +136,42 @@ protected void render(Block html) {
|
|||||||
.th(".nodeManagerVersion", "Version")
|
.th(".nodeManagerVersion", "Version")
|
||||||
.__().__().tbody();
|
.__().__().tbody();
|
||||||
|
|
||||||
// Add nodes to the web UI
|
if (nodesInfo != null && CollectionUtils.isNotEmpty(nodesInfo.getNodes())) {
|
||||||
for (NodeInfo info : nodes.getNodes()) {
|
for (NodeInfo info : nodesInfo.getNodes()) {
|
||||||
int usedMemory = (int) info.getUsedMemory();
|
int usedMemory = (int) info.getUsedMemory();
|
||||||
int availableMemory = (int) info.getAvailableMemory();
|
int availableMemory = (int) info.getAvailableMemory();
|
||||||
TR<TBODY<TABLE<Hamlet>>> row = tbody.tr();
|
TR<TBODY<TABLE<Hamlet>>> row = tbody.tr();
|
||||||
row.td().__(StringUtils.join(",", info.getNodeLabels())).__();
|
row.td().__(StringUtils.join(",", info.getNodeLabels())).__();
|
||||||
row.td().__(info.getRack()).__();
|
row.td().__(info.getRack()).__();
|
||||||
row.td().__(info.getState()).__();
|
row.td().__(info.getState()).__();
|
||||||
row.td().__(info.getNodeId()).__();
|
row.td().__(info.getNodeId()).__();
|
||||||
boolean isInactive = false;
|
boolean isInactive = false;
|
||||||
if (isInactive) {
|
if (isInactive) {
|
||||||
row.td().__("N/A").__();
|
row.td().__(UNAVAILABLE).__();
|
||||||
} else {
|
} else {
|
||||||
String httpAddress = info.getNodeHTTPAddress();
|
String httpAddress = info.getNodeHTTPAddress();
|
||||||
row.td().a("//" + httpAddress, httpAddress).__();
|
String herfWebAppAddress = "";
|
||||||
|
if (httpAddress != null && !httpAddress.isEmpty()) {
|
||||||
|
herfWebAppAddress =
|
||||||
|
WebAppUtils.getHttpSchemePrefix(this.router.getConfig()) + httpAddress;
|
||||||
|
}
|
||||||
|
row.td().a(herfWebAppAddress, httpAddress).__();
|
||||||
|
}
|
||||||
|
|
||||||
|
row.td().br().$title(String.valueOf(info.getLastHealthUpdate())).__()
|
||||||
|
.__(new Date(info.getLastHealthUpdate())).__()
|
||||||
|
.td(info.getHealthReport())
|
||||||
|
.td(String.valueOf(info.getNumContainers())).td().br()
|
||||||
|
.$title(String.valueOf(usedMemory)).__()
|
||||||
|
.__(StringUtils.byteDesc(usedMemory * BYTES_IN_MB)).__().td().br()
|
||||||
|
.$title(String.valueOf(availableMemory)).__()
|
||||||
|
.__(StringUtils.byteDesc(availableMemory * BYTES_IN_MB)).__()
|
||||||
|
.td(String.valueOf(info.getUsedVirtualCores()))
|
||||||
|
.td(String.valueOf(info.getAvailableVirtualCores()))
|
||||||
|
.td(info.getVersion()).__();
|
||||||
}
|
}
|
||||||
row.td().br().$title(String.valueOf(info.getLastHealthUpdate())).__()
|
|
||||||
.__(Times.format(info.getLastHealthUpdate())).__()
|
|
||||||
.td(info.getHealthReport())
|
|
||||||
.td(String.valueOf(info.getNumContainers())).td().br()
|
|
||||||
.$title(String.valueOf(usedMemory)).__()
|
|
||||||
.__(StringUtils.byteDesc(usedMemory * BYTES_IN_MB)).__().td().br()
|
|
||||||
.$title(String.valueOf(availableMemory)).__()
|
|
||||||
.__(StringUtils.byteDesc(availableMemory * BYTES_IN_MB)).__()
|
|
||||||
.td(String.valueOf(info.getUsedVirtualCores()))
|
|
||||||
.td(String.valueOf(info.getAvailableVirtualCores()))
|
|
||||||
.td(info.getVersion()).__();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
tbody.__().__();
|
tbody.__().__();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -18,7 +18,7 @@
|
|||||||
|
|
||||||
package org.apache.hadoop.yarn.server.router.webapp;
|
package org.apache.hadoop.yarn.server.router.webapp;
|
||||||
|
|
||||||
import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_STATE;
|
import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_SC;
|
||||||
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
|
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
|
||||||
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
|
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
|
||||||
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
|
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
|
||||||
@ -31,7 +31,7 @@ class NodesPage extends RouterView {
|
|||||||
@Override
|
@Override
|
||||||
protected void preHead(Page.HTML<__> html) {
|
protected void preHead(Page.HTML<__> html) {
|
||||||
commonPreHead(html);
|
commonPreHead(html);
|
||||||
String type = $(NODE_STATE);
|
String type = $(NODE_SC);
|
||||||
String title = "Nodes of the cluster";
|
String title = "Nodes of the cluster";
|
||||||
if (type != null && !type.isEmpty()) {
|
if (type != null && !type.isEmpty()) {
|
||||||
title = title + " (" + type + ")";
|
title = title + " (" + type + ")";
|
||||||
|
@ -33,16 +33,23 @@
|
|||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
import java.util.Collection;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.Comparator;
|
import java.util.Comparator;
|
||||||
|
|
||||||
public abstract class RouterBlock extends HtmlBlock {
|
public abstract class RouterBlock extends HtmlBlock {
|
||||||
|
|
||||||
private final Router router;
|
private final Router router;
|
||||||
|
private final ViewContext ctx;
|
||||||
|
private final FederationStateStoreFacade facade;
|
||||||
|
private final Configuration conf;
|
||||||
|
|
||||||
public RouterBlock(Router router, ViewContext ctx) {
|
public RouterBlock(Router router, ViewContext ctx) {
|
||||||
super(ctx);
|
super(ctx);
|
||||||
|
this.ctx = ctx;
|
||||||
this.router = router;
|
this.router = router;
|
||||||
|
this.facade = FederationStateStoreFacade.getInstance();
|
||||||
|
this.conf = this.router.getConfig();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -51,7 +58,6 @@ public RouterBlock(Router router, ViewContext ctx) {
|
|||||||
* @return Router ClusterMetricsInfo.
|
* @return Router ClusterMetricsInfo.
|
||||||
*/
|
*/
|
||||||
protected ClusterMetricsInfo getRouterClusterMetricsInfo() {
|
protected ClusterMetricsInfo getRouterClusterMetricsInfo() {
|
||||||
Configuration conf = this.router.getConfig();
|
|
||||||
boolean isEnabled = isYarnFederationEnabled();
|
boolean isEnabled = isYarnFederationEnabled();
|
||||||
if(isEnabled) {
|
if(isEnabled) {
|
||||||
String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf);
|
String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf);
|
||||||
@ -60,6 +66,7 @@ protected ClusterMetricsInfo getRouterClusterMetricsInfo() {
|
|||||||
.genericForward(webAppAddress, null, ClusterMetricsInfo.class, HTTPMethods.GET,
|
.genericForward(webAppAddress, null, ClusterMetricsInfo.class, HTTPMethods.GET,
|
||||||
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null,
|
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null,
|
||||||
conf, client);
|
conf, client);
|
||||||
|
client.destroy();
|
||||||
return metrics;
|
return metrics;
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
@ -72,7 +79,7 @@ protected ClusterMetricsInfo getRouterClusterMetricsInfo() {
|
|||||||
* @throws YarnException if the call to the getSubClusters is unsuccessful.
|
* @throws YarnException if the call to the getSubClusters is unsuccessful.
|
||||||
*/
|
*/
|
||||||
protected List<SubClusterInfo> getSubClusterInfoList() throws YarnException {
|
protected List<SubClusterInfo> getSubClusterInfoList() throws YarnException {
|
||||||
FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance();
|
|
||||||
Map<SubClusterId, SubClusterInfo> subClustersInfo = facade.getSubClusters(true);
|
Map<SubClusterId, SubClusterInfo> subClustersInfo = facade.getSubClusters(true);
|
||||||
|
|
||||||
// Sort the SubClusters.
|
// Sort the SubClusters.
|
||||||
@ -90,10 +97,81 @@ protected List<SubClusterInfo> getSubClusterInfoList() throws YarnException {
|
|||||||
* @return true, enable yarn federation; false, not enable yarn federation;
|
* @return true, enable yarn federation; false, not enable yarn federation;
|
||||||
*/
|
*/
|
||||||
protected boolean isYarnFederationEnabled() {
|
protected boolean isYarnFederationEnabled() {
|
||||||
Configuration conf = this.router.getConfig();
|
|
||||||
boolean isEnabled = conf.getBoolean(
|
boolean isEnabled = conf.getBoolean(
|
||||||
YarnConfiguration.FEDERATION_ENABLED,
|
YarnConfiguration.FEDERATION_ENABLED,
|
||||||
YarnConfiguration.DEFAULT_FEDERATION_ENABLED);
|
YarnConfiguration.DEFAULT_FEDERATION_ENABLED);
|
||||||
return isEnabled;
|
return isEnabled;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get a list of SubClusterIds for ActiveSubClusters.
|
||||||
|
*
|
||||||
|
* @return list of SubClusterIds.
|
||||||
|
*/
|
||||||
|
protected List<String> getActiveSubClusterIds() {
|
||||||
|
List<String> result = new ArrayList<>();
|
||||||
|
try {
|
||||||
|
Map<SubClusterId, SubClusterInfo> subClustersInfo = facade.getSubClusters(true);
|
||||||
|
subClustersInfo.values().stream().forEach(subClusterInfo -> {
|
||||||
|
result.add(subClusterInfo.getSubClusterId().getId());
|
||||||
|
});
|
||||||
|
} catch (Exception e) {
|
||||||
|
LOG.error("getActiveSubClusters error.", e);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* init SubCluster MetricsOverviewTable.
|
||||||
|
*
|
||||||
|
* @param html HTML Object.
|
||||||
|
* @param subclusterId subClusterId
|
||||||
|
*/
|
||||||
|
protected void initSubClusterMetricsOverviewTable(Block html, String subclusterId) {
|
||||||
|
MetricsOverviewTable metricsOverviewTable = new MetricsOverviewTable(this.router, this.ctx);
|
||||||
|
metricsOverviewTable.render(html, subclusterId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get ClusterMetricsInfo By SubClusterId.
|
||||||
|
*
|
||||||
|
* @param subclusterId subClusterId
|
||||||
|
* @return SubCluster RM ClusterMetricsInfo
|
||||||
|
*/
|
||||||
|
protected ClusterMetricsInfo getClusterMetricsInfoBySubClusterId(String subclusterId) {
|
||||||
|
try {
|
||||||
|
SubClusterId subClusterId = SubClusterId.newInstance(subclusterId);
|
||||||
|
SubClusterInfo subClusterInfo = facade.getSubCluster(subClusterId);
|
||||||
|
if (subClusterInfo != null) {
|
||||||
|
Client client = RouterWebServiceUtil.createJerseyClient(this.conf);
|
||||||
|
// Call the RM interface to obtain schedule information
|
||||||
|
String webAppAddress = WebAppUtils.getHttpSchemePrefix(this.conf) +
|
||||||
|
subClusterInfo.getRMWebServiceAddress();
|
||||||
|
ClusterMetricsInfo metrics = RouterWebServiceUtil
|
||||||
|
.genericForward(webAppAddress, null, ClusterMetricsInfo.class, HTTPMethods.GET,
|
||||||
|
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null,
|
||||||
|
conf, client);
|
||||||
|
client.destroy();
|
||||||
|
return metrics;
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
LOG.error("getClusterMetricsInfoBySubClusterId subClusterId = {} error.", subclusterId, e);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected Collection<SubClusterInfo> getSubClusterInfoList(String subclusterId) {
|
||||||
|
try {
|
||||||
|
SubClusterId subClusterId = SubClusterId.newInstance(subclusterId);
|
||||||
|
SubClusterInfo subClusterInfo = facade.getSubCluster(subClusterId);
|
||||||
|
return Collections.singletonList(subClusterInfo);
|
||||||
|
} catch (Exception e) {
|
||||||
|
LOG.error("getSubClusterInfoList subClusterId = {} error.", subclusterId, e);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public FederationStateStoreFacade getFacade() {
|
||||||
|
return facade;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -24,6 +24,8 @@
|
|||||||
import org.apache.hadoop.yarn.webapp.WebApp;
|
import org.apache.hadoop.yarn.webapp.WebApp;
|
||||||
import org.apache.hadoop.yarn.webapp.YarnWebParams;
|
import org.apache.hadoop.yarn.webapp.YarnWebParams;
|
||||||
|
|
||||||
|
import static org.apache.hadoop.yarn.util.StringHelper.pajoin;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The Router webapp.
|
* The Router webapp.
|
||||||
*/
|
*/
|
||||||
@ -48,7 +50,7 @@ public void setup() {
|
|||||||
route("/cluster", RouterController.class, "about");
|
route("/cluster", RouterController.class, "about");
|
||||||
route("/about", RouterController.class, "about");
|
route("/about", RouterController.class, "about");
|
||||||
route("/apps", RouterController.class, "apps");
|
route("/apps", RouterController.class, "apps");
|
||||||
route("/nodes", RouterController.class, "nodes");
|
route(pajoin("/nodes", NODE_SC), RouterController.class, "nodes");
|
||||||
route("/federation", RouterController.class, "federation");
|
route("/federation", RouterController.class, "federation");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -35,6 +35,9 @@ public class RouterClusterMetrics {
|
|||||||
protected static final long BYTES_IN_MB = 1024 * 1024;
|
protected static final long BYTES_IN_MB = 1024 * 1024;
|
||||||
private static final Logger LOG = LoggerFactory.getLogger(RouterClusterMetrics.class);
|
private static final Logger LOG = LoggerFactory.getLogger(RouterClusterMetrics.class);
|
||||||
|
|
||||||
|
// webPageTitlePrefix
|
||||||
|
private String webPageTitlePrefix = "Federation";
|
||||||
|
|
||||||
// Application Information.
|
// Application Information.
|
||||||
private String appsSubmitted = "N/A";
|
private String appsSubmitted = "N/A";
|
||||||
private String appsCompleted = "N/A";
|
private String appsCompleted = "N/A";
|
||||||
@ -99,6 +102,12 @@ public RouterClusterMetrics(ClusterMetricsInfo metrics) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public RouterClusterMetrics(ClusterMetricsInfo metrics,
|
||||||
|
String webPageTitlePrefix) {
|
||||||
|
this(metrics);
|
||||||
|
this.webPageTitlePrefix = webPageTitlePrefix;
|
||||||
|
}
|
||||||
|
|
||||||
// Get Key Metric Information
|
// Get Key Metric Information
|
||||||
public String getAppsSubmitted() {
|
public String getAppsSubmitted() {
|
||||||
return appsSubmitted;
|
return appsSubmitted;
|
||||||
@ -307,4 +316,8 @@ public void conversionNodeInformation(ClusterMetricsInfo metrics) {
|
|||||||
LOG.error("conversionNodeInformation error.", e);
|
LOG.error("conversionNodeInformation error.", e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public String getWebPageTitlePrefix() {
|
||||||
|
return webPageTitlePrefix;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -63,4 +63,22 @@ public void testFederationAboutViewNotEnable()
|
|||||||
config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, false);
|
config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, false);
|
||||||
WebAppTests.testPage(AboutPage.class, Router.class, new MockRouter(config));
|
WebAppTests.testPage(AboutPage.class, Router.class, new MockRouter(config));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testFederationNodeViewEnable()
|
||||||
|
throws InterruptedException, YarnException, IOException {
|
||||||
|
// Test Federation Enabled
|
||||||
|
Configuration config = new YarnConfiguration();
|
||||||
|
config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
|
||||||
|
WebAppTests.testPage(NodesPage.class, Router.class, new MockRouter(config));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testFederationNodeViewNotEnable()
|
||||||
|
throws InterruptedException, YarnException, IOException {
|
||||||
|
// Test Federation Not Enabled
|
||||||
|
Configuration config = new YarnConfiguration();
|
||||||
|
config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, false);
|
||||||
|
WebAppTests.testPage(NodesPage.class, Router.class, new MockRouter(config));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user