big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-11345. [Federation] Refactoring Yarn Router's Application Web Page. (#5030)

Browse Source
This commit is contained in:
slfan1989 2022-10-25 00:35:16 +08:00 committed by GitHub
parent 833750f72a
commit 454157a384
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 178 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
View File

@ -35,6 +35,7 @@ public interface YarnWebParams {
String APP_STATE = "app.state";
String APP_START_TIME_BEGIN = "app.started-time.begin";
String APP_START_TIME_END = "app.started-time.end";
String APP_SC = "app.subcluster";
String APPS_NUM = "apps.num";
String QUEUE_NAME = "queue.name";
String NODE_STATE = "node.state";

178
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java
View File

@ -21,11 +21,18 @@
import static org.apache.commons.text.StringEscapeUtils.escapeHtml4;
import static org.apache.commons.text.StringEscapeUtils.escapeEcmaScript;
import static org.apache.hadoop.yarn.util.StringHelper.join;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_SC;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE;
import com.sun.jersey.api.client.Client;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo;
@ -34,34 +41,102 @@
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TABLE;
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.TBODY;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import com.google.inject.Inject;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* Applications block for the Router Web UI.
*/
public class AppsBlock extends HtmlBlock {
public class AppsBlock extends RouterBlock {
private final Router router;
private final Configuration conf;
@Inject
AppsBlock(Router router, ViewContext ctx) {
super(ctx);
super(router, ctx);
this.router = router;
this.conf = this.router.getConfig();
}
@Override
protected void render(Block html) {
// Get the applications from the Resource Managers
Configuration conf = this.router.getConfig();
Client client = RouterWebServiceUtil.createJerseyClient(conf);
String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf);
AppsInfo apps = RouterWebServiceUtil
.genericForward(webAppAddress, null, AppsInfo.class, HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS, null, null, conf,
client);
setTitle("Applications");
boolean isEnabled = isYarnFederationEnabled();
// Get subClusterName
String subClusterName = $(APP_SC);
String reqState = $(APP_STATE);
// We will try to get the subClusterName.
// If the subClusterName is not empty,
// it means that we need to get the Node list of a subCluster.
AppsInfo appsInfo = null;
if (subClusterName != null && !subClusterName.isEmpty()) {
initSubClusterMetricsOverviewTable(html, subClusterName);
appsInfo = getSubClusterAppsInfo(subClusterName, reqState);
} else {
// Metrics Overview Table
html.__(MetricsOverviewTable.class);
appsInfo = getYarnFederationAppsInfo(isEnabled);
}
initYarnFederationAppsOfCluster(appsInfo, html);
}
private static String escape(String str) {
return escapeEcmaScript(escapeHtml4(str));
}
private AppsInfo getYarnFederationAppsInfo(boolean isEnabled) {
if (isEnabled) {
String webAddress = WebAppUtils.getRouterWebAppURLWithScheme(this.conf);
return getSubClusterAppsInfoByWebAddress(webAddress, StringUtils.EMPTY);
}
return null;
}
private AppsInfo getSubClusterAppsInfo(String subCluster, String states) {
try {
SubClusterId subClusterId = SubClusterId.newInstance(subCluster);
FederationStateStoreFacade facade = FederationStateStoreFacade.getInstance();
SubClusterInfo subClusterInfo = facade.getSubCluster(subClusterId);
if (subClusterInfo != null) {
// Prepare webAddress
String webAddress = subClusterInfo.getRMWebServiceAddress();
String herfWebAppAddress = "";
if (webAddress != null && !webAddress.isEmpty()) {
herfWebAppAddress = WebAppUtils.getHttpSchemePrefix(conf) + webAddress;
return getSubClusterAppsInfoByWebAddress(herfWebAppAddress, states);
}
}
} catch (Exception e) {
LOG.error("get AppsInfo From SubCluster = {} error.", subCluster, e);
}
return null;
}
private AppsInfo getSubClusterAppsInfoByWebAddress(String webAddress, String states) {
Client client = RouterWebServiceUtil.createJerseyClient(conf);
Map<String, String[]> queryParams = new HashMap<>();
if (StringUtils.isNotBlank(states)) {
queryParams.put("states", new String[]{states});
}
AppsInfo apps = RouterWebServiceUtil
.genericForward(webAddress, null, AppsInfo.class, HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.APPS, null, queryParams, conf,
client);
client.destroy();
return apps;
}
private void initYarnFederationAppsOfCluster(AppsInfo appsInfo, Block html) {
TBODY<TABLE<Hamlet>> tbody = html.table("#apps").thead()
.tr()
@ -81,45 +156,18 @@ protected void render(Block html) {
// Render the applications
StringBuilder appsTableData = new StringBuilder("[\n");
for (AppInfo app : apps.getApps()) {
try {
String percent = String.format("%.1f", app.getProgress() * 100.0F);
String trackingURL =
app.getTrackingUrl() == null ? "#" : app.getTrackingUrl();
// AppID numerical value parsed by parseHadoopID in yarn.dt.plugins.js
appsTableData.append("[\"")
.append("<a href='").append(trackingURL).append("'>")
.append(app.getAppId()).append("</a>\",\"")
.append(escape(app.getUser())).append("\",\"")
.append(escape(app.getName())).append("\",\"")
.append(escape(app.getApplicationType())).append("\",\"")
.append(escape(app.getQueue())).append("\",\"")
.append(String.valueOf(app.getPriority())).append("\",\"")
.append(app.getStartTime()).append("\",\"")
.append(app.getFinishTime()).append("\",\"")
.append(app.getState()).append("\",\"")
.append(app.getFinalStatus()).append("\",\"")
// Progress bar
.append("<br title='").append(percent).append("'> <div class='")
.append(C_PROGRESSBAR).append("' title='")
.append(join(percent, '%')).append("'> ").append("<div class='")
.append(C_PROGRESSBAR_VALUE).append("' style='")
.append(join("width:", percent, '%')).append("'> </div> </div>")
// History link
.append("\",\"<a href='").append(trackingURL).append("'>")
.append("History").append("</a>");
appsTableData.append("\"],\n");
if (appsInfo != null && CollectionUtils.isNotEmpty(appsInfo.getApps())) {
} catch (Exception e) {
LOG.info(
"Cannot add application {}: {}", app.getAppId(), e.getMessage());
List<String> appInfoList =
appsInfo.getApps().stream().map(this::parseAppInfoData).collect(Collectors.toList());
if (CollectionUtils.isNotEmpty(appInfoList)) {
String formattedAppInfo = StringUtils.join(appInfoList, ",");
appsTableData.append(formattedAppInfo);
}
}
if (appsTableData.charAt(appsTableData.length() - 2) == ',') {
appsTableData.delete(appsTableData.length() - 2,
appsTableData.length() - 1);
}
appsTableData.append("]");
html.script().$type("text/javascript")
.__("var appsTableData=" + appsTableData).__();
@ -127,7 +175,39 @@ protected void render(Block html) {
tbody.__().__();
}
private static String escape(String str) {
return escapeEcmaScript(escapeHtml4(str));
private String parseAppInfoData(AppInfo app) {
StringBuilder appsDataBuilder = new StringBuilder();
try {
String percent = String.format("%.1f", app.getProgress() * 100.0F);
String trackingURL = app.getTrackingUrl() == null ? "#" : app.getTrackingUrl();
// AppID numerical value parsed by parseHadoopID in yarn.dt.plugins.js
appsDataBuilder.append("[\"")
.append("<a href='").append(trackingURL).append("'>")
.append(app.getAppId()).append("</a>\",\"")
.append(escape(app.getUser())).append("\",\"")
.append(escape(app.getName())).append("\",\"")
.append(escape(app.getApplicationType())).append("\",\"")
.append(escape(app.getQueue())).append("\",\"")
.append(app.getPriority()).append("\",\"")
.append(app.getStartTime()).append("\",\"")
.append(app.getFinishTime()).append("\",\"")
.append(app.getState()).append("\",\"")
.append(app.getFinalStatus()).append("\",\"")
// Progress bar
.append("<br title='").append(percent).append("'> <div class='")
.append(C_PROGRESSBAR).append("' title='")
.append(join(percent, '%')).append("'> ").append("<div class='")
.append(C_PROGRESSBAR_VALUE).append("' style='")
.append(join("width:", percent, '%')).append("'> </div> </div>")
// History link
.append("\",\"<a href='").append(trackingURL).append("'>")
.append("History").append("</a>");
appsDataBuilder.append("\"]\n");
} catch (Exception e) {
LOG.warn("Cannot add application {}: {}", app.getAppId(), e.getMessage());
}
return appsDataBuilder.toString();
}
}

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsPage.java
View File

@ -20,11 +20,13 @@
import static org.apache.hadoop.yarn.util.StringHelper.sjoin;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_SC;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.yarn.webapp.SubView;
class AppsPage extends RouterView {
@ -37,9 +39,14 @@ protected void preHead(Page.HTML<__> html) {
setTableStyles(html, "apps", ".queue {width:6em}", ".ui {width:8em}");
// Set the correct title.
String subClusterName = $(APP_SC);
String reqState = $(APP_STATE);
reqState = (reqState == null || reqState.isEmpty() ? "All" : reqState);
setTitle(sjoin(reqState, "Applications"));
if(StringUtils.isBlank(subClusterName)){
subClusterName = "Federation ";
}
reqState = (StringUtils.isBlank(reqState) ? "All" : reqState);
setTitle(sjoin(subClusterName, reqState, "Applications"));
}
private String appsTableInit() {

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java
View File

@ -20,12 +20,12 @@
import com.google.inject.Inject;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.server.router.Router;
import org.apache.hadoop.yarn.server.webapp.WebPageUtils;
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
import java.util.List;
/**
* Navigation block for the Router Web UI.
*/
@ -60,7 +60,24 @@ public void render(Block html) {
subAppsList1.__().__();
// ### applications info
mainList.li().a(url("apps"), "Applications").__();
Hamlet.UL<Hamlet.LI<Hamlet.UL<Hamlet.DIV<Hamlet>>>> subAppsList2 =
mainList.li().a(url("apps"), "Applications").ul();
subAppsList2.li().__();
for (String subClusterId : subClusterIds) {
Hamlet.LI<Hamlet.UL<Hamlet.LI<Hamlet.UL<Hamlet.DIV<Hamlet>>>>> subAppsList3 = subAppsList2.
li().a(url("apps", subClusterId), subClusterId);
Hamlet.UL<Hamlet.LI<Hamlet.UL<Hamlet.LI<Hamlet.UL<Hamlet.DIV<Hamlet>>>>>> subAppsList4 =
subAppsList3.ul().$style("padding:0.3em 1em 0.1em 2em");
subAppsList4.li().__();
for (YarnApplicationState state : YarnApplicationState.values()) {
subAppsList4.
li().a(url("apps", subClusterId, state.toString()), state.toString()).__();
}
subAppsList4.li().__().__();
subAppsList3.__();
}
subAppsList2.__().__();
// ### tools
Hamlet.DIV<Hamlet> sectionBefore = mainList.__();

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java
View File

@ -116,6 +116,7 @@ private NodesInfo getSubClusterNodesInfoByWebAddress(String webAddress) {
.genericForward(webAddress, null, NodesInfo.class, HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.NODES, null, null, conf,
client);
client.destroy();
return nodes;
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebApp.java
View File

@ -49,7 +49,7 @@ public void setup() {
route("/", RouterController.class);
route("/cluster", RouterController.class, "about");
route("/about", RouterController.class, "about");
route("/apps", RouterController.class, "apps");
route(pajoin("/apps", APP_SC, APP_STATE), RouterController.class, "apps");
route(pajoin("/nodes", NODE_SC), RouterController.class, "nodes");
route("/federation", RouterController.class, "federation");
}

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationWebApp.java
View File

@ -81,4 +81,22 @@ public void testFederationNodeViewNotEnable()
config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, false);
WebAppTests.testPage(NodesPage.class, Router.class, new MockRouter(config));
}
@Test
public void testFederationAppViewEnable()
throws InterruptedException, YarnException, IOException {
// Test Federation Enabled
Configuration config = new YarnConfiguration();
config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
WebAppTests.testPage(AppsPage.class, Router.class, new MockRouter(config));
}
@Test
public void testFederationAppViewNotEnable()
throws InterruptedException, YarnException, IOException {
// Test Federation Not Enabled
Configuration config = new YarnConfiguration();
config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, false);
WebAppTests.testPage(AppsPage.class, Router.class, new MockRouter(config));
}
}